def can_connect(self):
SocketSession.remove_expired()
if models.SocketSession.objects(ip=self.environ.get('REMOTE_ADDR')).count() >= config.server_max_connection_per_ip:
error_msg = 'You have reached the limit of %d open sockets per IP' % config.server_max_connection_per_ip
self.emit('error', error_msg)
self.logger.debug('Disconnected: %s', error_msg)
return False
def can_connect(self):
SocketSession.remove_expired()
if models.SocketSession.objects(ip=self.environ.get('REMOTE_ADDR')).count() >= config.server_max_connection_per_ip:
error_msg = 'You have reached the limit of %d open sockets per IP' % config.server_max_connection_per_ip
self.emit('error', error_msg)
self.logger.debug('Disconnected: %s', error_msg)
return False
def __init__(self, terminate):
DataStreamAbstract.__init__(self, terminate)
self.stream = TwitterStreamHandler()
self.kw_track = SocketSession.get_keywords()
self.kw_hash = SocketSession.get_keywords_hash()
self.kw_int = config.twitter_kw_interval_check
self.kw_last_check = time.time()
def __init__(self, terminate):
DataStreamAbstract.__init__(self, terminate)
self.stream = TwitterStreamHandler()
self.kw_track = SocketSession.get_keywords()
self.kw_hash = SocketSession.get_keywords_hash()
self.kw_int = config.twitter_kw_interval_check
self.kw_last_check = time.time()
def __init__(self, terminate):
DataStreamAbstract.__init__(self, terminate)
self.auth = twitter.OAuth(config.twitter_oauth_token, config.twitter_oauth_secret,
config.twitter_oauth_custkey, config.twitter_oauth_custsecret)
self.kw_track = SocketSession.get_keywords()
self.kw_hash = SocketSession.get_keywords_hash()
self.kw_int = config.twitter_kw_interval_check
self.kw_last_check = time.time()
def check_keywords(self):
"""
check if twitter keywords are changed
"""
if (time.time() - self.kw_last_check) > self.kw_int:
kw_hash = SocketSession.get_keywords_hash()
if self.kw_hash != kw_hash:
self.kw_hash = kw_hash
self.kw_track = SocketSession.get_keywords()
raise TwitterWorkerKwChange('keywords change detected')
def check_keywords(self):
"""
check if twitter keywords are changed
"""
if (time.time() - self.kw_last_check) > self.kw_int:
kw_hash = SocketSession.get_keywords_hash()
if self.kw_hash != kw_hash:
self.kw_hash = kw_hash
self.kw_track = SocketSession.get_keywords()
raise TwitterWorkerKwChange('keywords change detected')
def test_start(self):
"""
test datastream worker pool
"""
s = SocketSession(ip='x')
s.keywords = ['google', 'bieber']
s.save()
w = WorkerPool()
w.start()
time.sleep(1)
w.terminate()
self.assertGreater(RawStreamQueue.objects.count(), 0)
def test_workerSleep(self):
"""
twitter collector should sleep if not keywords found
"""
RawStreamQueue.drop_collection()
SocketSession.drop_collection()
kill = threading.Event()
self.assertEqual(len(SocketSession.get_keywords()), 0)
w = TwitterWorker(kill)
w.setDaemon(True)
w.start()
time.sleep(1)
kill.set()
self.assertEqual(RawStreamQueue.objects.count(), 0)
def test_workerSleep(self):
"""
twitter collector should sleep if not keywords found
"""
RawStreamQueue.drop_collection()
SocketSession.drop_collection()
kill = threading.Event()
self.assertEqual(len(SocketSession.get_keywords()),0)
w = TwitterWorker(kill)
w.setDaemon(True)
w.start()
time.sleep(1)
kill.set()
self.assertEqual(RawStreamQueue.objects.count(), 0)
def test_start(self):
"""
test datastream worker pool
"""
s = SocketSession(ip='x')
s.keywords = ['google', 'bieber']
s.save()
w = WorkerPool()
w.start()
while not RawStreamQueue.objects.count():
time.sleep(0.1)
w.terminate()
self.assertGreater(RawStreamQueue.objects.count(), 0)
def test_worker(self):
"""
test single twitter worker
"""
s = SocketSession(ip='x')
s.keywords = ['google','bieber']
s.save()
kill = threading.Event()
w = TwitterWorker(kill)
w.setDaemon(True)
w.start()
# sleep is needed in order to get at least couple of tweets
time.sleep(2)
kill.set()
self.assertGreater(RawStreamQueue.objects.count(), 0)
def test_worker(self):
"""
test single twitter worker
"""
s = SocketSession(ip='x')
s.keywords = ['google','bieber']
s.save()
kill = threading.Event()
w = TwitterWorker(kill)
w.setDaemon(True)
w.start()
while not RawStreamQueue.objects.count():
time.sleep(0.1)
kill.set()
self.assertGreater(RawStreamQueue.objects.count(), 0)
def test_worker(self):
"""
test single twitter worker
"""
s = SocketSession(ip='x')
s.keywords = ['google', 'bieber']
s.save()
kill = threading.Event()
w = TwitterWorker(kill)
w.setDaemon(True)
w.start()
while not RawStreamQueue.objects.count():
time.sleep(0.1)
kill.set()
self.assertGreater(RawStreamQueue.objects.count(), 0)
def tearDown(self):
RawStreamQueue.drop_collection()
SocketSession.drop_collection()
def tearDown(self):
RawStreamQueue.drop_collection()
SocketSession.drop_collection()
