def test_default_anonymous_no_rate_limit(
client, client_ip, switch_on, mocker, url):
def fake_switch(name, default=True):
if name == SWITCH_ENABLE_RATE_LIMITER:
return switch_on
return default
mocker.patch.object(switch, 'is_switched_on', fake_switch)
if not switch_on:
mocker.patch.object(settings, 'RATE_LIMITER_SETTINGS', {
'__default__': {
'rate': 1,
'capacity': 3,
}
})
def worker(queue):
response = client.get(url)
if response.status_code == 429:
queue.put(True)
greenlets = []
queue = Queue()
for _ in range(3):
greenlets.append(gevent.spawn(worker, queue))
gevent.joinall(greenlets)
with raises(Empty):
queue.get_nowait()
def test_logged_concurrent_limit_with_redis_error(
client, client_ip, test_user, test_token, mocker,
error_method, cause_limit):
mocker.patch.object(settings, 'CONCURRENT_LIMITER_SETTINGS', {
test_user.username: {
'ttl': 100,
'capacity': 1,
}
})
mocker.patch.object(redis_client, error_method, side_effect=Exception)
def worker(queue):
response = client.get(
'/api/busy_with_login', headers={
'Authorization': test_token,
})
if response.status_code == 429:
queue.put(429)
greenlets = []
queue = Queue()
for _ in range(3):
greenlets.append(gevent.spawn(worker, queue))
gevent.joinall(greenlets)
if cause_limit:
assert queue.get_nowait() == 429
else:
with raises(Empty):
queue.get_nowait()
def test_default_logged_no_concurrent_limit(
client, client_ip, test_token, switch_on, mocker):
def fake_switch(name, default=True):
if name == SWITCH_ENABLE_CONCURRENT_LIMITER:
return switch_on
return default
mocker.patch.object(switch, 'is_switched_on', fake_switch)
if not switch_on:
mocker.patch.object(settings, 'CONCURRENT_LIMITER_SETTINGS', {
'__default__': {
'ttl': 100,
'capacity': 1,
}
})
def worker(queue):
response = client.get(
'/api/busy_with_login', headers={
'Authorization': test_token,
})
if response.status_code == 429:
queue.put(429)
greenlets = []
queue = Queue()
for _ in range(3):
greenlets.append(gevent.spawn(worker, queue))
gevent.joinall(greenlets)
with raises(Empty):
queue.get_nowait()
def test_logged_no_rate_limit_because_redis_error(
client, client_ip, test_user, test_token, mocker, error_method):
mocker.patch.object(settings, 'RATE_LIMITER_SETTINGS', {
test_user.username: {
'rate': 1,
'capacity': 3,
}
})
mocker.patch.object(redis_client, error_method, side_effect=Exception)
def worker(queue):
response = client.get(
'/api/need_login', headers={
'Authorization': test_token,
})
if response.status_code == 429:
queue.put(429)
greenlets = []
queue = Queue()
for _ in range(3):
greenlets.append(gevent.spawn(worker, queue))
gevent.joinall(greenlets)
with raises(Empty):
queue.get_nowait()
def test_logged_no_concurrent_limit_because_remain_count(
client, client_ip, test_user, test_token, mocker):
mocker.patch.object(settings, 'CONCURRENT_LIMITER_SETTINGS', {
test_user.username: {
'ttl': 100,
'capacity': 100,
}
})
def worker(queue):
response = client.get(
'/api/busy_with_login', headers={
'Authorization': test_token,
})
if response.status_code == 429:
queue.put(429)
greenlets = []
queue = Queue()
for _ in range(3):
greenlets.append(gevent.spawn(worker, queue))
gevent.joinall(greenlets)
with raises(Empty):
queue.get_nowait()
def test_logged_with_rate_limit(
client, client_ip, mocker, test_user, test_token,
configs, use_username):
if '127.0.0.1' in configs:
configs[client_ip] = configs.pop('127.0.0.1')
if use_username:
configs.update({
test_user.username: {
'rate': 1,
'capacity': 3,
}
})
mocker.patch.object(settings, 'RATE_LIMITER_SETTINGS', configs)
def worker(queue):
response = client.get('/api/need_login', headers={
'Authorization': test_token,
})
if response.status_code == 429:
queue.put(429)
greenlets = []
queue = Queue()
for _ in range(10):
greenlets.append(gevent.spawn(worker, queue))
gevent.joinall(greenlets)
assert queue.get_nowait() == 429
class LocalRPCServer(Greenlet):
def __init__(self, server_queue):
Greenlet.__init__(self)
self.server_queue = server_queue
self.recv_queue = Queue()
self.handle_stoping = False
self.recv_stop_evt = Event()
def write(self, callInfo):
self.recv_queue.put(callInfo)
# 对RPC请求队列中的请求进行处理
def recv_call_handle(self):
while True:
if self.recv_queue.empty() & self.handle_stoping:
self.recv_stop_evt.set()
return
if not self.recv_queue.empty():
callInfo = self.recv_queue.get_nowait()
callInfo.agent = self
self.server_queue.put(callInfo)
gevent.sleep(0)
def Callback(self, callInfo):
reply_to = callInfo.props["reply_to"]
reply_to.put(callInfo.Result)
def _run(self):
try:
self.recv_call_handle()
except Exception, e:
print e
class EventLoop():
def __init__(self, strategy):
self.strategy = strategy
self.event_queue = Queue()
def proc_strategy(self):
global DELAY
gevent.sleep(MAX_DELAY)
status = self.strategy.run()
if status == StrategyResult.SUCCESS:
DELAY = max(MIN_DELAY, DELAY / 2)
elif status == StrategyResult.FAILURE:
self.event_queue.put(EQType.STOP_EXECUTION)
elif status == StrategyResult.PASSED:
DELAY = min(MAX_DELAY, DELAY * 2)
def run(self):
pool = Pool(config.max_concurrency)
while True:
pool.spawn(self.proc_strategy)
try:
next_op = self.event_queue.get_nowait()
if next_op == EQType.STOP_EXECUTION:
pool.join()
break
except gevent.queue.Empty:
pass
class Crawler(object): def __init__(self, processor): self.processor = processor self.pool = Pool(self.processor.concurrency) self.base_host = urlsplit(self.processor.start_url).hostname self.urls = Queue() self.urls.put(self.processor.start_url) self.visited_urls = set() self.visited_urls_lock = RLock() self.pages_count = 0 def start(self): while True: if self.pages_count >= self.processor.max_pages: self.urls = Queue() break try: url = self.urls.get_nowait() self.pool.wait_available() spider = Spider(self, self.processor, url) self.pool.start(spider) self.pages_count += 1 except Empty: break self.pool.join() if not self.urls.empty(): self.start()
def test_login_with_concurrent_limit(
client, client_ip, mocker, test_user, test_token,
configs, use_username):
if '127.0.0.1' in configs:
configs[client_ip] = configs.pop('127.0.0.1')
if use_username:
configs.update({
test_user.username: {
'ttl': 100,
'capacity': 1,
}
})
mocker.patch.object(settings, 'CONCURRENT_LIMITER_SETTINGS', configs)
def worker(queue):
response = client.get('/api/busy_with_login', headers={
'Authorization': test_token,
})
if response.status_code != 200:
queue.put(response.status_code)
greenlets = []
queue = Queue()
for _ in range(3):
greenlets.append(gevent.spawn(worker, queue))
gevent.joinall(greenlets)
assert queue.get_nowait() == 429
def _run(self):
utils.log("[%s] parsing site %s" % (self, self.base))
queue = Queue()
pool = Pool(64)
seed = 'http://www.nytimes.com/best-sellers-books/'
pool.spawn(self._parseResultsPage, pool, queue, seed, 'current', True)
while True:
items = []
while not queue.empty():
item = queue.get_nowait()
items.append(item)
if 0 == len(items) and 0 == len(pool):
break
for item in items:
pool.spawn(item[0], pool, queue, item[1], item[2], item[3])
time.sleep(0.01)
pool.join()
self._output.put(StopIteration)
def _run(self):
utils.log("[%s] parsing site %s" % (self, self.base))
queue = Queue()
pool = Pool(64)
seed = 'http://community.seattletimes.nwsource.com/entertainment/i_results.php?search=venue&type=Restaurant&page=1'
pool.spawn(self._parseResultsPage, pool, queue, seed, '1', True)
while True:
items = []
while not queue.empty():
item = queue.get_nowait()
items.append(item)
if 0 == len(items) and 0 == len(pool):
break
for item in items:
pool.spawn(item[0], pool, queue, item[1], item[2], item[3])
time.sleep(0.01)
pool.join()
self._output.put(StopIteration)
def _run(self):
utils.log("[%s] parsing site %s" % (self, self.base))
queue = Queue()
pool = Pool(32)
seed = 'http://www.amazon.com/best-sellers-books-Amazon/zgbs/books/'
parsed = set()
queue.put_nowait((seed, 'seed', 0))
while True:
items = []
while not queue.empty():
item = queue.get_nowait()
if item[0] not in parsed:
items.append(item)
parsed.add(item[0])
if 0 == len(items) and 0 == len(pool):
break
for item in items:
pool.spawn(self._parseResultsPage, queue, item[0], item[1], item[2])
time.sleep(0.01)
pool.join()
self._output.put(StopIteration)
def _run(self):
utils.log("[%s] parsing site %s" % (self, self.base))
queue = Queue()
pool = Pool(16)
seed = 'http://www.awardannals.com/skin/menubar81.html'
pool.spawn(self._parseIndexPage, pool, queue, seed, 'index')
while True:
items = []
while not queue.empty():
item = queue.get_nowait()
items.append(item)
if 0 == len(items) and 0 == len(pool):
break
for item in items:
pool.spawn(self._parseResultsPage, pool, queue, item[0], item[1], False)
time.sleep(0.01)
pool.join()
self._output.put(StopIteration)
class __AudioNode(gevent.Greenlet):
RATE = 44100
CHUNK = 512
PORT = 20000
def __init__(self, is_log=True):
gevent.Greenlet.__init__(self)
self.is_log = is_log
self.command = Queue()
def is_quit(self):
try:
cmd = self.command.get_nowait()
return cmd == "q"
except Empty:
return False
def stop(self, msg=""):
self.command.put("q")
if self.is_log:
print "%s - stopping %s" % (self.__class__.__name__, msg)
def _run(self):
if self.is_log:
print "starting: %s" % self.__class__.__name__
try:
self.engine()
finally:
self.pa.close()
self.sock.close()
class BaseMailSyncMonitor(Greenlet):
def __init__(self, account_id, email_address, provider, status_cb,
heartbeat=1):
self.inbox = Queue()
# how often to check inbox, in seconds
self.heartbeat = heartbeat
self.log = configure_mailsync_logging(account_id)
self.account_id = account_id
self.email_address = email_address
self.provider = provider
# Stuff that might be updated later and we want to keep a shared
# reference on child greenlets.
if hasattr(self, 'shared_state'):
self.shared_state['status_cb'] = status_cb
else:
self.shared_state = dict(status_cb=status_cb)
Greenlet.__init__(self)
def _run(self):
return log_uncaught_errors(self._run_impl, self.log)()
def _run_impl(self):
sync = Greenlet.spawn(log_uncaught_errors(self.sync, self.log))
while not sync.ready():
try:
cmd = self.inbox.get_nowait()
if not self.process_command(cmd):
# ctrl-c, basically!
self.log.info("Stopping sync for {0}".format(
self.email_address))
# make sure the parent can't start/stop any folder monitors
# first
sync.kill(block=True)
killall(self.folder_monitors)
return
except Empty:
sleep(self.heartbeat)
assert not sync.successful(), \
"mail sync for {} account {} should run forever!"\
.format(self.provider, self.account_id)
raise sync.exception
def process_command(self, cmd):
""" Returns True if successful, or False if process should abort. """
self.log.info("processing command {0}".format(cmd))
return cmd != 'shutdown'
def _thread_finished(self, thread):
state = getattr(thread, 'state')
return state == 'finish'
def _thread_polling(self, thread):
state = getattr(thread, 'state')
return state is not None and state.startswith('poll')
def sync(self):
raise NotImplementedError
class QueueCommandManager(CommandManager):
"""
A command manager that takes a queue of functions that act on
the tracer and apply them one at a time with each call to next_command().
"""
def __init__(self, tracer):
super(QueueCommandManager, self).__init__(tracer)
self.queue = Queue()
self.sent = []
def enqueue(self, fn):
"""
Enqueues a new message to be consumed.
"""
self.queue.put(fn)
def user_wait(self, duration):
"""
Simulates waiting on the user to feed us input for duration seconds.
"""
self.enqueue(lambda t: gevent.sleep(duration))
def clear(self):
"""
Clears the internal list of functions.
"""
self.queue = Queue()
def user_next_command(self):
"""
Removes one message from the internal queue and apply it to the
debugger.
"""
try:
self.queue.get_nowait()(self.tracer)
except Empty:
return
def send(self, msg):
# Collect the output so that we can make assertions about it.
self.sent.append(pickle.loads(msg))
user_stop = clear
def start(self, auth_msg=''):
pass
class QueueCommandManager(CommandManager):
"""
A command manager that takes a queue of functions that act on
the tracer and apply them one at a time with each call to next_command().
"""
def __init__(self, tracer):
super(QueueCommandManager, self).__init__(tracer)
self.queue = Queue()
self.sent = []
def enqueue(self, fn):
"""
Enqueues a new message to be consumed.
"""
self.queue.put(fn)
def user_wait(self, duration):
"""
Simulates waiting on the user to feed us input for duration seconds.
"""
self.enqueue(lambda t: gevent.sleep(duration))
def clear(self):
"""
Clears the internal list of functions.
"""
self.queue = Queue()
def user_next_command(self):
"""
Removes one message from the internal queue and apply it to the
debugger.
"""
try:
self.queue.get_nowait()(self.tracer)
except Empty:
return
def send(self, msg):
# Collect the output so that we can make assertions about it.
self.sent.append(pickle.loads(msg))
user_stop = clear
def start(self, auth_msg=''):
pass
class MemorySession(Session):
"""
In memory session with a outgoing gevent Queue as the message
store.
"""
def __init__(self, server, session_id=None):
super(MemorySession, self).__init__(server, session_id=session_id)
self.session_id = session_id or str(uuid.uuid4())[:8]
self.server = server
self.queue = Queue()
self.hits = 0
self.heartbeats = 0
self.connected = False
def add_message(self, msg):
self.queue.put_nowait(msg)
def get_messages(self, **kwargs):
timeout = kwargs.get('timeout', None)
self.incr_hits()
if self.queue.empty():
try:
return self.queue.get(**kwargs)
except Empty:
return []
else:
accum = []
try:
while not self.queue.empty():
if timeout:
accum.append(self.queue.get(timeout=timeout))
else:
accum.append(self.queue.get_nowait())
finally:
return accum
def interrupt(self):
"""
A kill event trigged through a client accessible endpoint
Internal expires will not have is_interupted() == True
"""
self.interrupted = True
self.kill()
def kill(self):
self.connected = False
# Expire only once
if not self.expired:
self.expired = True
self.timeout.set()
class NonBlockingIMap(object):
''' NonBlockingIMap class
'''
def __init__(self, size=1, func=None, iterable=None):
''' __init__
'''
if size is not None and size <= 0:
raise ValueError('size must not be negative and not equal zero: %r' % (size, ))
self.size = size
if func is None:
raise RuntimeError('func must be assigned: %s' % (func, ))
self.func = func
self.greenlets_count = 0
self.iterable = iterable
self.queue = Queue()
def __iter__(self):
''' __iter__
'''
return self
if PY3:
__next__ = next
del next
def next(self):
''' next
'''
gevent.sleep()
value = None
if self.greenlets_count >= self.size:
return value
try:
item = self.iterable.get_nowait()
gevent.spawn(self.func, item).link(self._on_result)
self.greenlets_count += 1
except Empty:
pass
try:
value = self.queue.get_nowait()
except Empty:
pass
if not value and self.greenlets_count <= 0:
raise StopIteration()
return value
def _on_result(self, greenlet):
''' _on_result
'''
self.greenlets_count -= 1
if greenlet.successful():
self.queue.put(greenlet.value)
class WorkQueue(object):
def __init__(self, worker, start_runner=None, max_work_load=16):
if not worker:
# TODO raise exception
pass
self.worker = worker
self._start_runner = start_runner
self._max_work_load = max_work_load
# do we want to limit the size of the queue?
self._queue = Queue()
self._num_enqueues = 0
self._num_dequeues = 0
self._runner = Runner(self, self._max_work_load)
#end __init__
def enqueue(self, work_item):
self._queue.put(work_item)
self._num_enqueues = self._num_enqueues + 1
self.may_be_start_runner()
#end enqueue
def dequeue(self):
try:
work_item = self._queue.get_nowait()
except Empty:
work_item = None
else:
self._num_dequeues = self._num_dequeues + 1
return work_item
#end dequeue
def may_be_start_runner(self):
if self._queue.empty() or \
(self._start_runner and not self._start_runner()):
return
self._runner.start()
#end may_be_start_runner
def runner_done(self):
if self._queue.empty() or \
(self._start_runner and not self._start_runner()):
return True
return False
#end runner_done
def is_queue_empty(self):
if self._queue.empty():
return True
return False
def num_enqueues(self):
return self._num_enqueues
#end num_enqueues
def num_dequeues(self):
return self._num_dequeues
class AsyncPool(object):
__slots__ = ["sockets", "socket_factory", "pool_size", "log", "local", "pid" ]
def __init__(self, socket_factory, pool_size):
self.pid = os.getpid()
self.pool_size = pool_size
self.socket_factory = socket_factory
self.sockets = Queue()
self.log = logging.getLogger('ming.async.AsyncPool')
self.local = local()
def _get_sock(self):
return getattr(self.local, 'sock', None)
def _set_sock(self, value):
self.local.sock = value
sock = property(_get_sock, _set_sock)
def socket(self):
pid = os.getpid()
if pid != self.pid:
self.sock = None
self.sockets = Queue()
self.pid = pid
if self.sock is not None:
self.log.debug('Return existing socket to greenlet %s', gevent.getcurrent() )
return self.sock
gl = gevent.getcurrent()
try:
self.sock = self.sockets.get_nowait()
self.log.debug('Checkout socket %s to greenlet %s',
self.sock, gl )
except Empty:
self.sock = self.socket_factory()
self.log.debug('Create socket in greenlet %s', gl)
self.sock.last_greenlet = gl
return self.sock
def return_socket(self):
if self.sock is None:
self.log.debug('No socket to return from greenlet %s', gevent.getcurrent() )
return
if self.sockets.qsize() < self.pool_size:
gl = gevent.getcurrent()
self.log.debug('Checkin socket %s from greenlet %s',
self.sock, gl)
self.sockets.put(self.sock)
self.sock = None
else:
self.log.debug('Close socket in greenlet %s', gevent.getcurrent() )
self.sock.close()
self.sock = None
self.local.sock = None
class Pipeline(object):
"""Persistent, bounded-memory-usage pipeline:
- 'spills' items to disk if internal memory queue is filled-up.
- restores queue contents from a journal file upon restart after crash.
(planned, persistency is not implemented yet.)
"""
def __init__(self, receiver, logdir, size):
"""logdir: must exist
"""
self.receiver = receiver
self.makedata = getattr(self.receiver, 'makedata', lambda o: o)
self.logdir = logdir
self.size = size
self.queue = Queue(None)
self.items = None
self.seq = 0
def put(self, item):
# TODO: write to journal
data = self.makedata(item)
self.queue.put((self.seq, data))
def _get_batch(self):
# wait until at least one is available
items = [self.queue.get()]
# read more items if available
try:
while 1:
item = self.queue.get_nowait()
items.append(item)
except Empty:
pass
return items
def run(self):
try:
while 1:
if self.items is None:
self.items = self._get_batch()
if self.items:
try:
data = [o[1] for o in self.items]
self.receiver.put(data)
# TODO: write to journal
self.items = None
except Exception, ex:
logging.warn('%s.put() failed (%s), retrying...',
self.receiver,
ex,
exc_info=1)
gevent.sleep(5 * 60)
except Exception as ex:
logging.error('Pipeline.run exiting by error', exc_info=1)
class MemorySession(Session):
"""
In memory session with a outgoing gevent Queue as the message
store.
"""
timer = 10.0
def __init__(self, server, session_id=None):
super(MemorySession, self).__init__(server, session_id)
self.session_id = session_id or str(uuid.uuid4())[:8]
self.server = server
self.queue = Queue()
self.hits = 0
self.heartbeats = 0
self.connected = False
def add_message(self, msg):
self.queue.put_nowait(msg)
def get_messages(self, **kwargs):
self.incr_hits()
if self.queue.empty():
try:
return self.queue.get(**kwargs)
except Empty:
return []
else:
accum = []
try:
while not self.queue.empty():
accum.append(self.queue.get_nowait())
finally:
return accum
def interrupt(self):
"""
A kill event trigged through a client accessible endpoint
Internal expires will not have is_interupted() == True
"""
self.interrupted = True
self.kill()
def kill(self):
self.connected = False
# Expire only once
if not self.expired:
self.expired = True
self.timeout.set()
class geventimg(object):
def __init__(self):
self.file_path = 'D:/animeimg/animeimg2/'
self.queue = Queue()
def getanimeimgurlfrommysql(self):
try:
if not os.path.exists(self.file_path):
print ('文件夹',self.file_path,'不存在，重新建立')
os.makedirs(self.file_path)
db.execute('SELECT a.ANIME_ID,a.ANIME_IMAGE FROM anime_home a',None)
records = db.fetchall()
if records:
for r in records:
self.queue.put_nowait(r)
except Exception as e:
pass
else:
pass
finally:
pass
pass
def downloadimg(self,n):
while not self.queue.empty():
try:
animeimg = self.queue.get_nowait()
r = self.url_open('http://donghua.dmzj.com{}'.format(animeimg[1]))
# print(animeimg)
anime_img_name = self.file_path +'animepic_{}.jpg'.format(animeimg[0])
with open(anime_img_name,"wb") as f:
f.write(r.content)
f.flush()
f.close()
gevent.sleep(1)
except Exception as e:
traceback.print_exc()
finally:
pass
def url_open(self,url):
"""
爬取网页
"""
req = request.get(url,5,'donghua.dmzj.com',None)
return req
def main(self):
gevent.spawn(self.getanimeimgurlfrommysql).join()
# gevent.spawn(self.downloadimg).join()
pool = gevent.pool.Pool(5)
threads = []
for r in range(5):
threads.append(pool.spawn(self.downloadimg,r))
gevent.joinall(threads)
class ConnectionPool(object):
def __init__(self,
host,
port,
maxsize=10,
connect_timeout=None,
read_timeout=None,
factory=lambda x: x):
if not isinstance(maxsize, (int, long)):
raise TypeError('Expected integer, got %r' % (maxsize, ))
self.maxsize = maxsize
self.pool = Queue()
self.size = 0
self.host = host
self.port = port
self.factory = factory
self.connect_timeout = connect_timeout
self.read_timeout = read_timeout
def get(self):
pool = self.pool
if self.size >= self.maxsize or pool.qsize():
return pool.get()
else:
self.size += 1
try:
new_item = self.create_connection()
except:
self.size -= 1
raise
return new_item
def put(self, item):
self.pool.put(item)
def lose(self, item):
self.size -= 1
item.close()
def closeall(self):
while not self.pool.empty():
conn = self.pool.get_nowait()
try:
conn.close()
except Exception:
pass
def create_connection(self):
"""Create connection to remote host."""
sock = socket.create_connection((self.host, self.port),
timeout=self.connect_timeout)
sock.settimeout(self.read_timeout)
return self.factory(sock)
def test_anonymous_no_concurrent_limit_because_remain_count(
client, client_ip, mocker, url):
mocker.patch.object(settings, 'CONCURRENT_LIMITER_SETTINGS', {
'__anonymous__': {
'ttl': 100,
'capacity': 100,
}
})
def worker(queue):
response = client.get(url)
if response.status_code == 429:
queue.put(429)
greenlets = []
queue = Queue()
for _ in range(3):
greenlets.append(gevent.spawn(worker, queue))
gevent.joinall(greenlets)
with raises(Empty):
queue.get_nowait()
class Pipeline(object):
"""Persistent, bounded-memory-usage pipeline:
- 'spills' items to disk if internal memory queue is filled-up.
- restores queue contents from a journal file upon restart after crash.
(planned, persistency is not implemented yet.)
"""
def __init__(self, receiver, logdir, size):
"""logdir: must exist
"""
self.receiver = receiver
self.makedata = getattr(self.receiver, 'makedata', lambda o: o)
self.logdir = logdir
self.size = size
self.queue = Queue(None)
self.items = None
self.seq = 0
def put(self, item):
# TODO: write to journal
data = self.makedata(item)
self.queue.put((self.seq, data))
def _get_batch(self):
# wait until at least one is available
items = [self.queue.get()]
# read more items if available
try:
while 1:
item = self.queue.get_nowait()
items.append(item)
except Empty:
pass
return items
def run(self):
try:
while 1:
if self.items is None:
self.items = self._get_batch()
if self.items:
try:
data = [o[1] for o in self.items]
self.receiver.put(data)
# TODO: write to journal
self.items = None
except Exception, ex:
logging.warn('%s.put() failed (%s), retrying...',
self.receiver, ex, exc_info=1)
gevent.sleep(5*60)
except Exception as ex:
logging.error('Pipeline.run exiting by error', exc_info=1)
def test_socket_enqueues_incoming_messages():
queue = Queue()
request = testing.DummyRequest()
request.registry['streamer.work_queue'] = queue
socket = mock.Mock()
client = websocket.WebSocket(socket)
client.request = request
message = FakeMessage('client data')
client.received_message(message)
result = queue.get_nowait()
assert result.socket == client
assert result.payload == 'client data'
class ASyncHandler(Group):
def __init__(self, master, count=100, waitout=0.1):
super(ASyncHandler, self).__init__(
master, log=master.log, count=count, timeout=1)
self.master = master
self.queue = Queue()
self.waitout = waitout
self._size = 0
self._last = 0
def size(self):
if time.time() - self._last < 1:
return self._size
self._last = time.time()
self._size = self.queue.qsize()
return self._size
def do(self, func, *args, **kwargs):
if self.size() < 500:
self.queue.put((func, args, kwargs))
else:
func(*args, **kwargs)
def quick(self, limit=50):
count = 0
while count < limit:
try:
func, args, kwargs = self.queue.get_nowait()
func(*args, **kwargs)
count += 1
except Empty:
break
def handle(self):
count = 0
while True:
try:
func, args, kwargs = self.queue.get()
func(*args, **kwargs)
count += 1
except Empty:
break
self.wait(self.waitout)
def on_quit(self):
if not self.queue.empty():
self.log.info('%s - has %d tasks not done. now doing ...'
% (self.name, self.queue.qsize()))
def test_process_nsq_topic_connects_reader_on_message_to_handle_message(get_reader):
settings = {}
queue = Queue()
message = mock.Mock(body='hello')
reader = get_reader.return_value
reader.topic = 'donkeys'
nsq.process_nsq_topic(settings, 'donkeys', queue, raise_error=False)
message_handler = reader.on_message.connect.call_args[1]['receiver']
message_handler(reader, message)
result = queue.get_nowait()
assert result.topic == 'donkeys'
assert result.payload == 'hello'
def test_socket_enqueues_incoming_messages():
queue = Queue()
request = testing.DummyRequest()
request.registry['streamer.work_queue'] = queue
socket = mock.Mock()
client = websocket.WebSocket(socket)
client.request = request
message = FakeMessage('client data')
client.received_message(message)
result = queue.get_nowait()
assert result.socket == client
assert result.payload == 'client data'
class ConnectionPool(object):
def __init__(self, host, port, maxsize=10, connect_timeout=None,
read_timeout=None, factory=lambda x: x):
if not isinstance(maxsize, (int, long)):
raise TypeError('Expected integer, got %r' % (maxsize, ))
self.maxsize = maxsize
self.pool = Queue()
self.size = 0
self.host = host
self.port = port
self.factory = factory
self.connect_timeout = connect_timeout
self.read_timeout = read_timeout
def get(self):
pool = self.pool
if self.size >= self.maxsize or pool.qsize():
return pool.get()
else:
self.size += 1
try:
new_item = self.create_connection()
except Exception:
self.size -= 1
raise
return new_item
def put(self, item):
self.pool.put(item)
def lose(self, item):
self.size -= 1
item.close()
def closeall(self):
while not self.pool.empty():
conn = self.pool.get_nowait()
try:
conn.close()
except Exception:
pass
def create_connection(self):
"""Create connection to remote host."""
sock = socket.create_connection((self.host, self.port),
timeout=self.connect_timeout)
sock.settimeout(self.read_timeout)
return self.factory(sock)
def test_handle_recv_event_puts_data_in_queue(self, gevent_mock,
uwsgi_mock):
recv_event = Event()
recv_queue = Queue()
recv_event.set()
data = ['data1', 'data2', None]
uwsgi_mock.websocket_recv_nb.side_effect = data
self.handler.handle_recv_event(mock.Mock(), recv_event, recv_queue)
self.assertTrue(gevent_mock.spawn.called)
# Assert if recv_event is cleared
self.assertFalse(recv_event.is_set())
self.assertEqual(len(recv_queue), 2)
for d in data[:2]:
self.assertEqual(recv_queue.get_nowait(), d)
def test_process_nsq_topic_connects_reader_on_message_to_handle_message(
get_reader):
settings = {}
queue = Queue()
message = mock.Mock(body='hello')
reader = get_reader.return_value
reader.topic = 'donkeys'
nsq.process_nsq_topic(settings, 'donkeys', queue, raise_error=False)
message_handler = reader.on_message.connect.call_args[1]['receiver']
message_handler(reader, message)
result = queue.get_nowait()
assert result.topic == 'donkeys'
assert result.payload == 'hello'
class ConnectionPool(object):
def __init__(self, connection_cls, maxsize=100, **kwargs):
if not isinstance(maxsize, integer_types):
raise TypeError('Expected integer, got %r' % (maxsize, ))
self._connection_cls = connection_cls
self._maxsize = maxsize
self._pool = Queue()
self._size = 0
self._conn_params = kwargs
def get(self):
if self._size >= self._maxsize or self._pool.qsize():
return self._pool.get()
else:
self._size += 1
try:
return self._connection_cls(**self._conn_params)
except:
self._size -= 1
raise
def put(self, item):
self._pool.put(item)
def closeall(self):
while not self._pool.empty():
conn = self._pool.get_nowait()
try:
conn.close()
except Exception:
pass
@contextlib.contextmanager
def connection(self):
conn = self.get()
try:
yield conn
except:
if conn.closed:
conn = None
self.closeall()
raise
finally:
if conn is not None and not conn.closed:
self.put(conn)
def test_anonymous_with_rate_limit(client, client_ip, mocker, configs, url):
cfg = deepcopy(configs[0])
if '127.0.0.1' in cfg:
cfg[client_ip] = cfg.pop('127.0.0.1')
mocker.patch.object(settings, 'RATE_LIMITER_SETTINGS', cfg)
def worker(queue):
response = client.get(url)
if response.status_code == 429:
queue.put(429)
greenlets = []
queue = Queue()
for _ in range(5):
greenlets.append(gevent.spawn(worker, queue))
gevent.joinall(greenlets)
assert queue.get_nowait() == 429
class Node(object):
node_id = None
ip = None
port = None
pool = None
def __init__(self, node_id, ip, port):
self.node_id = node_id
self.ip = ip
self.port = port
self.pool = Queue()
@classmethod
def create(cls, ip, port):
return Node(uuid1(), ip, port)
def __hash__(self):
return long(self.node_id.hex, 16)
def __eq__(self, other):
return self.node_id == other.node_id
def send(self, message):
with self.connection as c:
c.send(message)
@property
@contextmanager
def connection(self):
# returns a connection
# to be used as
# with node.connection as c:
# do_stuff_with_c(c)
try:
connection = self.pool.get_nowait()
except Empty:
connection = Connection.connect(self.ip, self.port)
yield connection
self.pool.put(connection)
def event_stream(topic):
'''Publishes events based on the topic.'''
q = Queue()
subscriptions[topic].append(q)
try:
while True:
try:
result = q.get_nowait()
except gevent.queue.Empty:
gevent.sleep(1)
continue
ev = ServerSentEvent(str(result))
yield ev.encode()
except GeneratorExit:
subscriptions[topic].remove(q)
except Exception as ex:
print "Generic event-stream exception!"
print ex
subscriptions[topic].remove(q)
class DatabaseConnectionPool(object):
def __init__(self, maxsize=100):
if not isinstance(maxsize, (int, long)):
raise TypeError('Expected integer, got %r' % (maxsize, ))
self.maxsize = maxsize
self.pool = Queue()
self.size = 0
def get(self):
pool = self.pool
if self.size >= self.maxsize or pool.qsize():
new_item = pool.get()
try:
# check connection is still valid
self.check_usable(new_item)
logger.debug("DB connection reused")
except DatabaseError:
logger.debug("DB connection was closed, creating new one")
new_item = self.create_connection()
return new_item
else:
self.size += 1
try:
new_item = self.create_connection()
logger.debug("DB connection created")
except:
self.size -= 1
raise
return new_item
def put(self, item):
self.pool.put(item, timeout=2)
def closeall(self):
while not self.pool.empty():
conn = self.pool.get_nowait()
try:
conn.close()
except Exception:
pass
self.size = 0
class Actor(gevent.Greenlet):
def __init__(self):
self.inbox = Queue()
Greenlet.__init__(self)
def receive(self, message):
"""
Define in your subclass.
"""
raise NotImplemented()
def _run(self):
self.running = True
while self.running:
try:
message = self.inbox.get_nowait()
self.receive(message)
except Empty:
pass
gevent.sleep(0)
class MemorySession(session.Session):
"""
In memory session with a ``gevent.pool.Queue`` as the message store.
"""
def __init__(self, *args, **kwargs):
super(MemorySession, self).__init__(*args, **kwargs)
self._queue = Queue()
def add_message(self, frame, data):
log.info('session closed: %s', self.id)
self._queue.put_nowait((frame, data))
#waiter = self._waiter
#if waiter is not None:
# self._waiter = None
# if not waiter.cancelled():
# waiter.set_result(True)
self._tick()
def get_messages(self, timeout=None):
self._tick()
messages = []
# get all messages immediately pending in the queue
while not self._queue.empty():
try:
msg = self._queue.get_nowait()
except Empty:
break
messages.append(msg)
if not messages:
try:
messages.append(self._queue.get(timeout=timeout))
except Empty:
pass
return messages
def listen():
runtime = 0.0
rq = Queue(maxsize=None)
stop_event = gevent.event.Event()
gevent.spawn(
self.ipool.listen_on, result_queue=rq, channel_name="notify_test_values", cancel_event=stop_event
)
import time
while 1:
st = time.time()
if runtime > 5.0:
break
try:
notify = rq.get_nowait()
print "NOTIFY", notify
except QueueEmptyException:
gevent.sleep(0.001)
tt = time.time() - st
runtime += tt
stop_event.set()
gevent.sleep(1)
def listen():
runtime = 0.0
rq = Queue(maxsize=None)
stop_event = gevent.event.Event()
gevent.spawn(self.ipool.listen_on,
result_queue=rq,
channel_name='notify_test_values',
cancel_event=stop_event)
import time
while 1:
st = time.time()
if runtime > 5.0:
break
try:
notify = rq.get_nowait()
print "NOTIFY", notify
except QueueEmptyException:
gevent.sleep(0.001)
tt = time.time() - st
runtime += tt
stop_event.set()
gevent.sleep(1)
class BaseMailSyncMonitor(Greenlet):
"""
The SYNC_MONITOR_CLS for all mail sync providers should subclass this.
Parameters
----------
account_id : int
Which account to sync.
email_address : str
Email address for `account_id`.
provider : str
Provider for `account_id`.
heartbeat : int
How often to check for commands.
retry_fail_classes : list
Additional exceptions to *not* retry on. (This base class sets some
defaults.)
"""
RETRY_FAIL_CLASSES = [MailsyncError, ValueError, AttributeError, DataError,
IntegrityError, TypeError]
def __init__(self, account, heartbeat=1, retry_fail_classes=[]):
self.inbox = Queue()
# how often to check inbox, in seconds
self.heartbeat = heartbeat
self.log = log.new(component='mail sync', account_id=account.id)
self.account_id = account.id
self.email_address = account.email_address
self.provider_name = account.provider
self.retry_fail_classes = self.RETRY_FAIL_CLASSES
self.retry_fail_classes.extend(retry_fail_classes)
# Stuff that might be updated later and we want to keep a shared
# reference on child greenlets.
if not hasattr(self, 'shared_state'):
self.shared_state = dict()
Greenlet.__init__(self)
def _run(self):
return retry_and_report_killed(self._run_impl,
account_id=self.account_id,
logger=self.log,
fail_classes=self.retry_fail_classes)
def _run_impl(self):
sync = Greenlet(retry_and_report_killed, self.sync,
account_id=self.account_id, logger=self.log,
fail_classes=self.retry_fail_classes)
sync.start()
while not sync.ready():
try:
cmd = self.inbox.get_nowait()
if not self.process_command(cmd):
# ctrl-c, basically!
self.log.info("Stopping sync", email=self.email_address)
# make sure the parent can't start/stop any folder monitors
# first
sync.kill(block=True)
self.folder_monitors.kill()
return
except Empty:
sleep(self.heartbeat)
if sync.successful():
self.folder_monitors.kill()
return
# We just want the name of the exception so don't bother with
# sys.exc_info()
self.log.error('mail sync should run forever',
provider=self.provider_name,
account_id=self.account_id,
exception=type(sync.exception).__name__)
raise sync.exception
def process_command(self, cmd):
""" Returns True if successful, or False if process should abort. """
self.log.info("processing command {0}".format(cmd))
return cmd != 'shutdown'
def _thread_finished(self, thread):
state = getattr(thread, 'state')
return state == 'finish'
def _thread_polling(self, thread):
state = getattr(thread, 'state')
return state is not None and state.startswith('poll')
def sync(self):
raise NotImplementedError
__author__ = 'susperius'
import logging
import gevent
import gevent.monkey
from gevent.queue import Queue
import worker.fuzzingworker as dbgworker
from node.fuzzing.browser.javascript import JsDomFuzz
gevent.monkey.patch_all()
if __name__ == "__main__":
logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.DEBUG)
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
fuz = JsDomFuzz(10, 1000, "ie")
rep_q = Queue()
#dbg_w = dbgworker.DebuggerWorker("C:\\Program Files\\Tracker Software\\PDF Viewer\\PDFXcview.exe\" -t \"testcases\\test.pdf", fuz, rep_q, 5)
dbg_w = dbgworker.FuzzingWorker("C:\\Program Files\\Internet Explorer\\iexplore.exe", fuz, rep_q, 5, True)
dbg_w.start_worker()
gevent.sleep(20)
rep = rep_q.get_nowait()
print(rep)
dbg_w.__create_testcases()
class Actor(Greenlet):
"""This is the base class for every node in the process tree. `Actor` manages the children of the various process nodes."""
def __init__(self, name):
super(Actor, self).__init__()
self.inbox = Queue()
self.name = name
self.running = False
self._context = None
# stores results of node if required
# self._state = Message()
# child nodes
self._children = {}
# listeners
self.listeners = {}
# all 'normal' messages go through self.process
self.on(events.DATA, self.process)
# on node start, execute preProcess
self.on(events.START, self.preProcess)
self.on(events.START, lambda msg: setattr(self, 'running', True))
# on node stop, set running to false, flush the queue and execute postProcess
self.on(events.STOP, self.flush)
self.on(events.STOP, self.postProcess)
# self.on(events.STOP, lambda msg: setattr(self, 'running', False))
# self.on(events.STOP, self.kill)
# on kill, call postProcess
self.on(events.KILL, self.postProcess)
self.on(events.KILL, lambda msg: setattr(self, 'running', False))
# self.on(events.KILL, self.kill)
# listen to exceptions
self.link_exception(self.handleException)
def setContext(self, ctx):
"""Sets the execution context"""
ctx.addActor(self)
self._context = ctx
for child in self.children:
child.setContext(ctx)
def getContext(self):
"""Gets the execution context"""
return self._context
def tick(self):
"""Yields the event loop to another node"""
# self.log_trace("Yielding...")
gevent.sleep(0)
def sleep(self, seconds):
"""Makes the node sleep for the given seconds"""
self.log_trace("Sleeping %ss..." % seconds)
gevent.sleep(seconds)
def handleException(self, exc):
pass
def stop(self):
"""Stop self and children"""
self.handle(events.StopMessage)
# gevent.joinall(list(self.children))
# self.tick()
def start(self):
"""Starts the nodes"""
if not self.running:
self.handle(events.StartMessage)
for child in self.children:
child.start()
super(Actor, self).start()
# @property
# def state(self):
# """Returns the results from this processing element."""
# return self._state
def clear(self):
"""Removes all children from this node"""
self._children = {}
def __len__(self):
return len(self._children)
def add(self, node, lazy=False):
"""Adds a child to the current node."""
validateType("node", (Actor, SynchronousActor), node)
if not lazy:
if node.name in self._children:
raise Exception(
"Same name siblings are not allowed. Child node, %s, already exists."
% node.name)
self._children[node.name] = (len(self._children), node)
else:
name = str(node.name)
num = 0
while name in self:
num += 1
name = (node.name + "-" + str(num))
node.name = name
self._children[name] = (len(self._children), node)
return self
def __getitem__(self, name):
"""Returns the child with the name given. Raises NodeDoesNotExist exception if child does not exist."""
if not name in self._children:
raise exceptions.NodeDoesNotExist(name)
return self._children[name][1]
def __contains__(self, name):
"""
Returns True if the node has the particular node.
"""
return name in self._children
@property
def children(self):
"""Returns a list of chilren in the order they were added."""
return (element for index, element in sorted(self._children.values()))
def hasChildren(self):
"""Returns True if the node has any children."""
return len(self._children) > 0
def __delitem__(self, name):
"""Removes the child with the given name. Raises NodeDoesNotExist exception if the child does not exist."""
if not name in self._children:
raise exceptions.NodeDoesNotExist(name)
del self._children[name]
def __isub__(self, child):
del self[child]
return self
def __iadd__(self, child):
return self.add(child)
def __str__(self):
return "<%s: name=\"%s\">" % (self.__class__.__name__, self.name)
def __iter__(self):
return self.children
def preProcess(self, message=None):
"""Pre processes the data source. Can be overriden."""
pass
def process(self, message=None):
"""This is the function which generates the results."""
raise NotImplementedError("process")
def postProcess(self, message=None):
"""Post processes the data source. Can be overriden.
Assumes inbox has been flushed and should close any open files / connections.
"""
pass
def flush(self, message=None):
"""Empties the queue"""
# gevent.joinall(self.dependencies)
# waiting = True
# while waiting:
# waiting = False
# for dep in self.dependencies:
# if dep.running:
# waiting = True
# self.tick()
self.log_debug("Flushing Queue...")
if not self.inbox.empty():
for message in self.inbox:
self.handle(message)
self.running = False
def execute(self):
"""Executes the preProcess, process, postProcess, scatter and gather methods"""
self.running = True
self.log_info("Starting...")
while self.running:
# self.log("Running...")
try:
message = self.inbox.get_nowait()
self.handle(message)
# yield to event loop after processing the message
self.tick()
except Empty:
# Empty signifies that the queue is empty, so yield to another node
self.tick()
# pass
# self.tick()
# self.stop()
self.log_info("Exiting...")
def _run(self):
self.execute()
def scatter(self, message, forward=False):
"""Sends the results of this node to its children."""
# if self.hasChildren():
self.emit('data', message, forward)
def emit(self, channel, message, forward=False):
"""The `emit` function can be used to send 'out-of-band' messages to
any child nodes. This essentially allows a node to send special messages
to any nodes listening."""
validateType('message', Message, message)
message.channel = channel
message.forward = forward
# message.source = self
self.log_debug("Sending message: %s on channel: %s" %
(message, channel))
for child in self.children:
child.insert(message)
# child.inbox.put_nowait(message)
# yields to event loop
self.tick()
def emitBatch(self, channel, messages, forward=False):
"""Add all messages to child queues before yielding"""
for message in messages:
message.channel = channel
message.forward = forward
# message.source = self
self.log_debug("Sending message: %s on channel: %s" %
(message, channel))
for child in self.children:
# child.inbox.queue.extend(messages)
child.insert(message)
# for child in self.children:
# child.inbox.queue.extend(messages)
# yields to event loop
self.tick()
def removeListener(self, channel, function):
"""Removes a listener function from a channel"""
if channel in self.listeners:
try:
self.listeners[channel].remove(function)
return True
except ValueError:
return False
else:
return False
def on(self, channel, function):
"""Registers event listeners based on a channel.
Note: All functions registered should have 2 arguments. The first is current node (ie. self) and the second is the data being transferred.
"""
self.log_trace("Registering function on channel '%s'" % (channel))
if channel in self.listeners:
self.listeners[channel].append(function)
else:
self.listeners[channel] = [function]
def handle(self, message):
"""Handles events received on a given channel and forwards it if allowed."""
# logging message
# if self.verbose:
# self.log("Processing message: %s" % message)
if message.channel in self.listeners:
for function in self.listeners[message.channel]:
# print function
function(message)
# forwards message if allowed
if message.forward:
self.log_trace("Forwarding message on channel: %s" %
message.channel)
for child in self.children:
child.insert(message)
# child.handle(message)
# child.inbox.put_nowait(message)
# self.tick()
def insert(self, message):
"""Inserts a new message to be handled"""
self.inbox.put_nowait(message)
def log(self, msg):
"""Logging capability is baked into every Node."""
self.log_info(msg)
def log_debug(self, msg):
"""Logs a DEBUG message"""
log_debug(str(self.name).upper(), msg)
def log_info(self, msg):
"""Logs an INFO message"""
log_info(str(self.name).upper(), msg)
def log_warning(self, msg):
"""Logs a WARNING message"""
log_warning(str(self.name).upper(), msg)
def log_error(self, msg):
"""Logs an ERROR message"""
log_error(str(self.name).upper(), msg)
def log_critical(self, msg):
"""Logs a CRITICAL message"""
log_critical(str(self.name).upper(), msg)
def log_exception(self, msg):
"""Logs an exception"""
log_exception(str(self.name).upper(), msg)
def log_trace(self, msg):
"""Logs low-level debug message"""
log_trace(str(self.name).upper(), msg)
class Notifier(object):
def __init__(self, app=None):
self._url = None
self._channel = None
self._interval = 1
self._started = False
self._q = Queue()
self._logger = None
if app:
self.init_app(app)
def init_app(self, app):
self._url = app.config['SLACK_INCOMING_WEBHOOK']
self._channel = app.config['SLACK_CHANNEL']
self._interval = 1 / float(app.config['SLACK_RATE_LIMIT'])
self._logger = app.logger
spawn(self._loop)
app.extensions['slack_notifier'] = self
self._started = True
self._logger.info(
'slack notifier started. min interval: %.1fs; default channel: %s',
self._interval, self._channel)
def notify(self, msg, channel=None, now=False):
"""
send a message to slack.
The messages are buffered under the hood to avoid hitting Slack API
rate limit.
:param msg: message to send
:param channel: channel to send the message
"""
if self._started:
channel = channel if channel else self._channel
if now:
self._send(msg, channel)
else:
self._q.put((channel, msg))
def stop(self):
self._started = False
self.notify(id(self), None) # use id as token to stop
def _send(self, message, channel):
"""
send to slack incoming webhook
:param str message: message to send
:param str channel: channel to send the message in
:returns: True if send succeeds
"""
data = {
'text': message,
'parse': 'full',
}
if channel:
data['channel'] = channel
try:
r = requests.post(self._url, data=json.dumps(data))
except (requests.ConnectionError, requests.Timeout) as e:
self._logger.warning('Fail to send slack request: %s', e)
return False
else:
if r.status_code == 200:
return True
else:
self._logger.warning(
'Non-200 code returned from slack: %d - %s', r.status_code,
r.content)
return False
def _flush(self):
"""
flush current queuing messages
"""
togo = defaultdict(list)
try:
while True:
channel, msg = self._q.get_nowait()
togo[channel].append(msg)
except Empty:
pass
flushed = []
for channel, messages in togo.iteritems():
msg = '\n'.join(messages)
if not self._send(msg, channel):
self._logger.error('fail to send message to slack %s - %s',
channel, msg)
else:
flushed.append((channel, msg))
return flushed
def _loop(self):
"""
send loop
"""
last_send = time.time()
while True:
top = self._q.peek(block=True)
if top == id(self): # stop
break
interval = time.time() - last_send
if interval >= self._interval:
# flush queue
flushed = self._flush()
last_send = time.time()
self._logger.debug('flush finished at %.3f: %s', last_send,
flushed)
else:
sleep(interval)
self.notify('Bye.')
self._flush()
class DatabaseConnectionPool(object):
def __init__(self, maxsize=100):
if not isinstance(maxsize, (int, long)):
raise TypeError('Expected integer, got %r' % (maxsize, ))
self.maxsize = maxsize
self.pool = Queue()
self.size = 0
def get(self):
pool = self.pool
if self.size >= self.maxsize or pool.qsize():
return pool.get()
else:
self.size += 1
try:
new_item = self.create_connection()
except:
self.size -= 1
raise
return new_item
def put(self, item):
self.pool.put(item)
def closeall(self):
while not self.pool.empty():
conn = self.pool.get_nowait()
try:
conn.close()
except Exception:
pass
@contextlib.contextmanager
def connection(self, isolation_level=None):
conn = self.get()
try:
if isolation_level is not None:
if conn.isolation_level == isolation_level:
isolation_level = None
else:
conn.set_isolation_level(isolation_level)
yield conn
except:
if conn.closed:
conn = None
self.closeall()
else:
conn = self._rollback(conn)
raise
else:
if conn.closed:
raise OperationalError("Cannot commit because connection was closed: %r" % (conn, ))
conn.commit()
finally:
if conn is not None and not conn.closed:
if isolation_level is not None:
conn.set_isolation_level(isolation_level)
self.put(conn)
@contextlib.contextmanager
def cursor(self, *args, **kwargs):
isolation_level = kwargs.pop('isolation_level', None)
conn = self.get()
try:
if isolation_level is not None:
if conn.isolation_level == isolation_level:
isolation_level = None
else:
conn.set_isolation_level(isolation_level)
yield conn.cursor(*args, **kwargs)
except:
if conn.closed:
conn = None
self.closeall()
else:
conn = self._rollback(conn)
raise
else:
if conn.closed:
raise OperationalError("Cannot commit because connection was closed: %r" % (conn, ))
conn.commit()
finally:
if conn is not None and not conn.closed:
if isolation_level is not None:
conn.set_isolation_level(isolation_level)
self.put(conn)
def _rollback(self, conn):
try:
conn.rollback()
except:
gevent.get_hub().handle_error(conn, *sys.exc_info())
return
return conn
def execute(self, *args, **kwargs):
with self.cursor(**kwargs) as cursor:
cursor.execute(*args)
return cursor.rowcount
def fetchone(self, *args, **kwargs):
with self.cursor(**kwargs) as cursor:
cursor.execute(*args)
return cursor.fetchone()
def fetchall(self, *args, **kwargs):
with self.cursor(**kwargs) as cursor:
cursor.execute(*args)
return cursor.fetchall()
def fetchiter(self, *args, **kwargs):
with self.cursor(**kwargs) as cursor:
cursor.execute(*args)
while True:
items = cursor.fetchmany()
if not items:
break
for item in items:
yield item
class WorkQueue(object):
_MAX_QUEUE_SIZE = 1024
_MAX_WORKLOAD = 16
def __init__(self,
worker,
start_runner=None,
max_qsize=None,
max_work_load=None):
self.worker = worker
self._start_runner = start_runner
self._max_qsize = max_qsize or WorkQueue._MAX_QUEUE_SIZE
self._max_work_load = max_work_load or WorkQueue._MAX_WORKLOAD
self._bounded = False
self._queue = Queue()
self._qsize = 0
self._num_enqueues = 0
self._num_dequeues = 0
self._drops = 0
self._high_watermarks = None
self._low_watermarks = None
self._hwm_index = -1
self._lwm_index = -1
self._runner = Runner(self, self._max_work_load)
# end __init__
def set_bounded(self, bounded):
self._bounded = bounded
# end set_bounded
def bounded(self):
return self._bounded
# end bounded
def set_high_watermarks(self, high_wm):
# weed out duplicates and store the watermarks in sorted order
self._high_watermarks = list(sorted(set(high_wm)))
self._set_watermark_indices(-1, -1)
# end set_high_watermarks
def high_watermarks(self):
return self._high_watermarks
# end high_watermarks
def set_low_watermarks(self, low_wm):
# weed out duplicates and store the watermarks in sorted order
self._low_watermarks = list(sorted(set(low_wm)))
self._set_watermark_indices(-1, -1)
# end set_low_watermarks
def low_watermarks(self):
return self._low_watermarks
# end low_watermarks
def watermark_indices(self):
return self._hwm_index, self._lwm_index
# end watermark_indices
def enqueue(self, work_item):
self.increment_queue_size(work_item)
if self._bounded:
if self._qsize > self._max_qsize:
self.decrement_queue_size(work_item)
self._drops += 1
return False
self._num_enqueues += 1
self._process_high_watermarks()
self._queue.put(work_item)
self.may_be_start_runner()
return True
# end enqueue
def dequeue(self):
try:
work_item = self._queue.get_nowait()
except Empty:
work_item = None
else:
self.decrement_queue_size(work_item)
self._num_dequeues += 1
self._process_low_watermarks()
return work_item
# end dequeue
def increment_queue_size(self, work_item):
self._qsize += 1
# end increment_queue_size
def decrement_queue_size(self, work_item):
self._qsize -= 1
# end decrement_queue_size
def size(self):
return self._qsize
# end size
def may_be_start_runner(self):
if self._queue.empty() or \
(self._start_runner and not self._start_runner()):
return
self._runner.start()
# end may_be_start_runner
def runner_done(self):
if self._queue.empty() or \
(self._start_runner and not self._start_runner()):
return True
return False
# end runner_done
def is_queue_empty(self):
if self._queue.empty():
return True
return False
# end is_queue_empty
def num_enqueues(self):
return self._num_enqueues
# end num_enqueues
def num_dequeues(self):
return self._num_dequeues
# end num_dequeues
def drops(self):
return self._drops
# end drops
def runner(self):
return self._runner
# end runner
def _set_watermark_indices(self, hwm_index, lwm_index):
self._hwm_index = hwm_index
self._lwm_index = lwm_index
# end _set_watermark_indices
def _process_high_watermarks(self):
if not self._high_watermarks:
return
# Check if we have crossed any high watermarks.
# Find the index of the first element greater than self._qsize
# in self._high_watermarks.
index = bisect.bisect_right(self._high_watermarks,
WaterMark(self._qsize, None))
# If the first element > qsize, then we have not crossed any
# high watermark.
if index == 0:
return
# We have crossed (index-1)th watermark in the list.
hwm_index = index - 1
if hwm_index == self._hwm_index:
return
self._set_watermark_indices(hwm_index, hwm_index + 1)
# Now invoke the watermark callback
self._high_watermarks[self._hwm_index].callback(self._qsize)
# end _process_high_watermarks
def _process_low_watermarks(self):
if not self._low_watermarks:
return
# Check if we have crossed any low watermarks.
# Find the index of the first element not less than self._qsize
# in self._low_watermarks.
index = bisect.bisect_left(self._low_watermarks,
WaterMark(self._qsize, None))
# If there is no element >= qsize, then we have not crossed any
# low watermark.
if index == len(self._low_watermarks):
return
lwm_index = index
if lwm_index == self._lwm_index:
return
self._set_watermark_indices(lwm_index - 1, lwm_index)
# Now invoke the watermark callback
self._low_watermarks[self._lwm_index].callback(self._qsize)
class Connection(object):
in_buffer_size = 4096
out_buffer_size = 4096
cql_version = None
keyspace = None
compression = True
compressor = None
decompressor = None
ssl_options = None
last_error = None
in_flight = 0
is_defunct = False
is_closed = False
lock = None
def __init__(
self,
host="127.0.0.1",
port=9042,
credentials=None,
ssl_options=None,
sockopts=None,
compression=True,
cql_version=None,
):
self.host = host
self.port = port
self.credentials = credentials
self.ssl_options = ssl_options
self.sockopts = sockopts
self.compression = compression
self.cql_version = cql_version
self._id_queue = Queue(MAX_STREAM_PER_CONNECTION)
for i in range(MAX_STREAM_PER_CONNECTION):
self._id_queue.put_nowait(i)
self.lock = RLock()
def close(self):
raise NotImplementedError()
def defunct(self, exc):
with self.lock:
if self.is_defunct or self.is_closed:
return
self.is_defunct = True
trace = traceback.format_exc(exc)
if trace != "None":
log.debug("Defuncting connection (%s) to %s: %s\n%s", id(self), self.host, exc, traceback.format_exc(exc))
else:
log.debug("Defuncting connection (%s) to %s: %s", id(self), self.host, exc)
self.last_error = exc
self.close()
self.error_all_callbacks(exc)
self.connected_event.set()
return exc
def error_all_callbacks(self, exc):
with self.lock:
callbacks = self._callbacks
self._callbacks = {}
new_exc = ConnectionShutdown(str(exc))
for cb in callbacks.values():
try:
cb(new_exc)
except Exception:
log.warn(
"Ignoring unhandled exception while erroring callbacks for a " "failed connection (%s) to host %s:",
id(self),
self.host,
exc_info=True,
)
def handle_pushed(self, response):
log.debug("Message pushed from server: %r", response)
for cb in self._push_watchers.get(response.event_type, []):
try:
cb(response.event_args)
except Exception:
log.exception("Pushed event handler errored, ignoring:")
def send_msg(self, msg, cb, wait_for_id=False):
if self.is_defunct:
raise ConnectionShutdown("Connection to %s is defunct" % self.host)
elif self.is_closed:
raise ConnectionShutdown("Connection to %s is closed" % self.host)
if not wait_for_id:
try:
request_id = self._id_queue.get_nowait()
except Queue.Empty:
raise ConnectionBusy("Connection to %s is at the max number of requests" % self.host)
else:
request_id = self._id_queue.get()
self._callbacks[request_id] = cb
self.push(msg.to_string(request_id, compression=self.compressor))
return request_id
def wait_for_response(self, msg, timeout=None):
return self.wait_for_responses(msg, timeout=timeout)[0]
def wait_for_responses(self, *msgs, **kwargs):
timeout = kwargs.get("timeout")
waiter = ResponseWaiter(self, len(msgs))
# busy wait for sufficient space on the connection
messages_sent = 0
while True:
needed = len(msgs) - messages_sent
with self.lock:
available = min(needed, MAX_STREAM_PER_CONNECTION - self.in_flight)
self.in_flight += available
for i in range(messages_sent, messages_sent + available):
self.send_msg(msgs[i], partial(waiter.got_response, index=i), wait_for_id=True)
messages_sent += available
if messages_sent == len(msgs):
break
else:
if timeout is not None:
timeout -= 0.01
if timeout <= 0.0:
raise OperationTimedOut()
time.sleep(0.01)
try:
return waiter.deliver(timeout)
except OperationTimedOut:
raise
except Exception, exc:
self.defunct(exc)
raise
class Server(object):
def __init__(self, address, size=None, log_level=DEFAULT_LOG_LEVEL):
self.daemon = True
self.started = False
self.size = size
self.queue = Queue(maxsize=size)
self.address = address
self.context = zmq.Context(1)
self.server = None
self.logger = get_logger(self, log_level)
self._has_fetched_jobs = False
def send(self, cmd, data=''):
self.server.send_multipart([cmd, data])
def recv(self):
reply = self.server.recv_multipart()
assert len(reply) == 2
return reply
def bind(self):
if self.server:
self.server.close()
self.server = self.context.socket(zmq.REP)
self.server.bind(self.address)
def start(self):
self.started = True
self.logger.info("Taskmaster binding to %r", self.address)
self.bind()
while self.started:
gevent.sleep(0)
cmd, data = self.recv()
if cmd == 'GET':
if not self.has_work():
self.send('QUIT')
continue
try:
job = self.queue.get_nowait()
except Empty:
self.send('WAIT')
continue
self.send('OK', pickle.dumps(job))
elif cmd == 'DONE':
self.queue.task_done()
if self.has_work():
self.send('OK')
else:
self.send('QUIT')
else:
self.send('ERROR', 'Unrecognized command')
self.logger.info('Shutting down')
self.shutdown()
def mark_queue_filled(self):
self._has_fetched_jobs = True
def put_job(self, job):
return self.queue.put(job)
def first_job(self):
return self.queue.queue[0]
def get_current_size(self):
return self.queue.qsize()
def get_max_size(self):
return self.size
def has_work(self):
if not self._has_fetched_jobs:
return True
return not self.queue.empty()
def is_alive(self):
return self.started
def shutdown(self):
if not self.started:
return
self.server.close()
self.context.term()
self.started = False
class WebSocket:
def __init__(self, socket):
self.socket = socket
self.closed = False
self.status = None
self._receive_error = None
self._queue = Queue()
self.max_length = 10 * 1024 * 1024
gevent.spawn(self._listen)
def set_max_message_length(self, length):
self.max_length = length
def _listen(self):
try:
while True:
fin = False
message = bytearray()
is_first_message = True
start_opcode = None
while not fin:
payload, opcode, fin = self._get_frame(max_length=self.max_length - len(message))
# Make sure continuation frames have correct information
if not is_first_message and opcode != 0:
self._error(STATUS_PROTOCOL_ERROR)
if is_first_message:
if opcode not in (OPCODE_TEXT, OPCODE_BINARY):
self._error(STATUS_PROTOCOL_ERROR)
# Save opcode
start_opcode = opcode
message += payload
is_first_message = False
message = bytes(message)
if start_opcode == OPCODE_TEXT: # UTF-8 text
try:
message = message.decode()
except UnicodeDecodeError:
self._error(STATUS_DATA_ERROR)
self._queue.put(message)
except Exception as e:
self.closed = True
self._receive_error = e
self._queue.put(None) # To make sure the error is read
def receive(self):
if not self._queue.empty():
return self.receive_nowait()
if isinstance(self._receive_error, EOFError):
return None
if self._receive_error:
raise self._receive_error
self._queue.peek()
return self.receive_nowait()
def receive_nowait(self):
ret = self._queue.get_nowait()
if self._receive_error and not isinstance(self._receive_error, EOFError):
raise self._receive_error
return ret
def send(self, data):
if self.closed:
raise EOFError()
if isinstance(data, str):
self._send_frame(OPCODE_TEXT, data.encode())
elif isinstance(data, bytes):
self._send_frame(OPCODE_BINARY, data)
else:
raise TypeError("Expected str or bytes, got " + repr(type(data)))
# Reads a frame from the socket. Pings, pongs and close packets are handled
# automatically
def _get_frame(self, max_length):
while True:
payload, opcode, fin = self._read_frame(max_length=max_length)
if opcode == OPCODE_PING:
self._send_frame(OPCODE_PONG, payload)
elif opcode == OPCODE_PONG:
pass
elif opcode == OPCODE_CLOSE:
if len(payload) >= 2:
self.status = struct.unpack("!H", payload[:2])[0]
was_closed = self.closed
self.closed = True
if not was_closed:
# Send a close frame in response
self.close(STATUS_OK)
raise EOFError()
else:
return payload, opcode, fin
# Low-level function, use _get_frame instead
def _read_frame(self, max_length):
header = self._recv_exactly(2)
if not (header[1] & 0x80):
self._error(STATUS_POLICY_VIOLATION)
opcode = header[0] & 0xf
fin = bool(header[0] & 0x80)
payload_length = header[1] & 0x7f
if payload_length == 126:
payload_length = struct.unpack("!H", self._recv_exactly(2))[0]
elif payload_length == 127:
payload_length = struct.unpack("!Q", self._recv_exactly(8))[0]
# Control frames are handled in a special way
if opcode in (OPCODE_PING, OPCODE_PONG):
max_length = 125
if payload_length > max_length:
self._error(STATUS_TOO_LONG)
mask = self._recv_exactly(4)
payload = self._recv_exactly(payload_length)
payload = self._unmask(payload, mask)
return payload, opcode, fin
def _recv_exactly(self, length):
buf = bytearray()
while len(buf) < length:
block = self.socket.recv(min(4096, length - len(buf)))
if block == b"":
raise EOFError()
buf += block
return bytes(buf)
def _unmask(self, payload, mask):
def gen(c):
return bytes([x ^ c for x in range(256)])
payload = bytearray(payload)
payload[0::4] = payload[0::4].translate(gen(mask[0]))
payload[1::4] = payload[1::4].translate(gen(mask[1]))
payload[2::4] = payload[2::4].translate(gen(mask[2]))
payload[3::4] = payload[3::4].translate(gen(mask[3]))
return bytes(payload)
def _send_frame(self, opcode, data):
for i in range(0, len(data), SEND_PACKET_SIZE):
part = data[i:i + SEND_PACKET_SIZE]
fin = int(i == (len(data) - 1) // SEND_PACKET_SIZE * SEND_PACKET_SIZE)
header = bytes(
[
(opcode if i == 0 else 0) | (fin << 7),
min(len(part), 126)
]
)
if len(part) >= 126:
header += struct.pack("!H", len(part))
self.socket.sendall(header + part)
def _error(self, status):
self.close(status)
raise EOFError()
def close(self, status=STATUS_OK):
self.closed = True
try:
self._send_frame(OPCODE_CLOSE, struct.pack("!H", status))
except (BrokenPipeError, ConnectionResetError):
pass
self.socket.close()
class AbstractDatabaseConnectionPool(object):
def __init__(self, maxsize=100):
if not isinstance(maxsize, integer_types):
raise TypeError('Expected integer, got %r' % (maxsize, ))
self.maxsize = maxsize
self.pool = Queue()
self.size = 0
def create_connection(self):
raise NotImplementedError()
def get(self):
pool = self.pool
if self.size >= self.maxsize or pool.qsize():
return pool.get()
self.size += 1
try:
new_item = self.create_connection()
except:
self.size -= 1
raise
return new_item
def put(self, item):
self.pool.put(item)
def closeall(self):
while not self.pool.empty():
conn = self.pool.get_nowait()
try:
conn.close()
except Exception:
pass
@contextlib.contextmanager
def connection(self, isolation_level=None):
conn = self.get()
try:
if isolation_level is not None:
if conn.isolation_level == isolation_level:
isolation_level = None
else:
conn.set_isolation_level(isolation_level)
yield conn
except:
if conn.closed:
conn = None
self.closeall()
else:
conn = self._rollback(conn)
raise
else:
if conn.closed:
raise OperationalError(
"Cannot commit because connection was closed: %r" %
(conn, ))
conn.commit()
finally:
if conn is not None and not conn.closed:
if isolation_level is not None:
conn.set_isolation_level(isolation_level)
self.put(conn)
@contextlib.contextmanager
def cursor(self, *args, **kwargs):
isolation_level = kwargs.pop('isolation_level', None)
with self.connection(isolation_level) as conn:
yield conn.cursor(*args, **kwargs)
def _rollback(self, conn):
try:
conn.rollback()
except:
gevent.get_hub().handle_error(conn, *sys.exc_info())
return
return conn
def execute(self, *args, **kwargs):
with self.cursor(**kwargs) as cursor:
cursor.execute(*args)
return cursor.rowcount
def fetchone(self, *args, **kwargs):
with self.cursor(**kwargs) as cursor:
cursor.execute(*args)
return cursor.fetchone()
def fetchall(self, *args, **kwargs):
with self.cursor(**kwargs) as cursor:
cursor.execute(*args)
return cursor.fetchall()
def fetchiter(self, *args, **kwargs):
with self.cursor(**kwargs) as cursor:
cursor.execute(*args)
while True:
items = cursor.fetchmany()
if not items:
break
for item in items:
yield item
class DatabaseConnectionPool(object):
""" Gevent compliant database connection pool """
def __init__(self, maxsize=100):
if not isinstance(maxsize, (int, long)):
raise TypeError('Expected integer, got %r' % (maxsize, ))
self.maxsize = maxsize # Maximum connections (pool + checkout out)
self.pool = Queue() # Open connection pool
self.size = 0 # Number of open connections
def get(self):
pool = self.pool
if self.size >= self.maxsize or pool.qsize():
return pool.get()
else:
self.size += 1
try:
new_item = self.create_connection()
except:
self.size -= 1
raise
return new_item
def put(self, item):
self.pool.put(item)
def closeall(self):
while not self.pool.empty():
conn = self.pool.get_nowait()
try:
conn.close()
self.size -= 1
except Exception:
pass
@contextlib.contextmanager
def in_transaction(self, isolation_level=None):
trans_conn = getattr(db_context, "cur_transaction", None)
if trans_conn:
raise OperationalError("Already in a transaction context")
conn = self.get()
db_context.cur_transaction = conn
try:
if isolation_level is not None:
if conn.isolation_level == isolation_level:
isolation_level = None
else:
conn.set_isolation_level(isolation_level)
yield conn
except:
if conn.closed:
conn = None
self.closeall()
else:
conn = self._rollback(conn)
raise
else:
if conn.closed:
raise OperationalError(
"Cannot commit because connection was closed: %r" %
(conn, ))
conn.commit()
finally:
if conn is not None and not conn.closed:
if isolation_level is not None:
conn.set_isolation_level(isolation_level)
self.put(conn)
db_context.cur_transaction = None
@contextlib.contextmanager
def connection(self, isolation_level=None):
trans_conn = getattr(db_context, "cur_transaction", None)
conn = trans_conn if trans_conn else self.get()
try:
if isolation_level is not None:
if conn.isolation_level == isolation_level:
isolation_level = None
else:
conn.set_isolation_level(isolation_level)
yield conn
except:
if conn.closed:
conn = None
self.closeall()
else:
conn = self._rollback(conn)
raise
else:
if conn.closed:
raise OperationalError(
"Cannot commit because connection was closed: %r" %
(conn, ))
if not trans_conn:
conn.commit()
finally:
if conn is not None and not conn.closed:
if isolation_level is not None:
conn.set_isolation_level(isolation_level)
if not trans_conn:
self.put(conn)
@contextlib.contextmanager
def cursor(self, *args, **kwargs):
isolation_level = kwargs.pop('isolation_level', None)
trans_conn = getattr(db_context, "cur_transaction", None)
conn = trans_conn if trans_conn else self.get()
try:
if isolation_level is not None:
if conn.isolation_level == isolation_level:
isolation_level = None
else:
conn.set_isolation_level(isolation_level)
tracer = kwargs.pop("tracer", None)
cur = conn.cursor(*args, **kwargs)
if isinstance(cur, TracingCursor):
cur._tracer = tracer
yield cur
except:
if conn.closed:
conn = None
self.closeall()
else:
conn = self._rollback(conn)
raise
else:
if conn.closed:
raise OperationalError(
"Cannot commit because connection was closed: %r" %
(conn, ))
if not trans_conn:
conn.commit()
finally:
if conn is not None and not conn.closed:
if isolation_level is not None:
conn.set_isolation_level(isolation_level)
if not trans_conn:
self.put(conn)
def _rollback(self, conn):
try:
conn.rollback()
except:
gevent.get_hub().handle_error(conn, *sys.exc_info())
return
return conn
def execute(self, *args, **kwargs):
with self.cursor(**kwargs) as cursor:
cursor.execute(*args)
return cursor.rowcount
def fetchone(self, *args, **kwargs):
with self.cursor(**kwargs) as cursor:
cursor.execute(*args)
return cursor.fetchone()
def fetchall(self, *args, **kwargs):
with self.cursor(**kwargs) as cursor:
cursor.execute(*args)
return cursor.fetchall()
def fetchiter(self, *args, **kwargs):
with self.cursor(**kwargs) as cursor:
cursor.execute(*args)
while True:
items = cursor.fetchmany()
if not items:
break
for item in items:
yield item
class Actor(object):
"""
Class that contains a queue and a greenlet serving that queue.
"""
max_ops_before_yield = 10000
"""Number of calls to self._maybe_yield before it yields"""
def __init__(self, qualifier=None):
self._event_queue = Queue()
self.greenlet = gevent.Greenlet(self._loop)
self._op_count = 0
self._current_msg = None
self.started = False
# Message being processed; purely for logging.
self.msg_uuid = None
# Logging parameters
self.qualifier = qualifier
if qualifier:
self.name = "%s(%s)" % (self.__class__.__name__, qualifier)
else:
self.name = self.__class__.__name__
# Can't use str(self) yet, it might not be ready until subclass
# constructed.
_log.info("%s created.", self.name)
def start(self):
assert not self.greenlet, "Already running"
_log.info("Starting %s", self)
self.started = True
self.greenlet.start()
return self
def _loop(self):
"""
Main greenlet loop, repeatedly runs _step(). Doesn't return normally.
"""
actor_storage.name = self.name
actor_storage.msg_uuid = None
try:
while True:
self._step()
except:
_log.exception("Exception killed %s", self)
raise
def _step(self):
"""
Run one iteration of the event loop for this actor. Mainly
broken out to allow the UTs to single-step an Actor.
It also has the beneficial side effect of introducing a new local
scope so that our variables die before we block next time.
"""
# Block waiting for work.
msg = self._event_queue.get()
actor_storage.msg_uuid = msg.uuid
batch = [msg]
batches = []
if not msg.needs_own_batch:
# Try to pull some more work off the queue to combine into a
# batch.
while not self._event_queue.empty():
# We're the only ones getting from the queue so this should
# never fail.
msg = self._event_queue.get_nowait()
if msg.needs_own_batch:
if batch:
batches.append(batch)
batches.append([msg])
batch = []
else:
batch.append(msg)
if batch:
batches.append(batch)
num_splits = 0
while batches:
# Process the first batch on our queue of batches. Invariant:
# we'll either process this batch to completion and discard it or
# we'll put all the messages back into the batch queue in the same
# order but with a first batch that is half the size and the
# rest of its messages in the second batch.
batch = batches.pop(0)
# Give subclass a chance to filter the batch/update its state.
batch = self._start_msg_batch(batch)
assert batch is not None, "_start_msg_batch() should return batch."
results = [] # Will end up same length as batch.
for msg in batch:
_log.debug("Message %s recd by %s from %s, queue length %d",
msg, msg.recipient, msg.caller,
self._event_queue.qsize())
self._current_msg = msg
try:
# Actually execute the per-message method and record its
# result.
result = msg.method()
except BaseException as e:
_log.exception("Exception processing %s", msg)
results.append(ResultOrExc(None, e))
else:
results.append(ResultOrExc(result, None))
finally:
self._current_msg = None
try:
# Give subclass a chance to post-process the batch.
_log.debug("Finishing message batch")
self._finish_msg_batch(batch, results)
except SplitBatchAndRetry:
# The subclass couldn't process the batch as is (probably
# because a failure occurred and it couldn't figure out which
# message caused the problem). Split the batch into two and
# re-run it.
_log.warn("Splitting batch to retry.")
self.__split_batch(batch, batches)
num_splits += 1 # For diags.
continue
except BaseException as e:
# Most-likely a bug. Report failure to all callers.
_log.exception("_finish_msg_batch failed.")
results = [(None, e)] * len(results)
# Batch complete and finalized, set all the results.
assert len(batch) == len(results)
for msg, (result, exc) in zip(batch, results):
for future in msg.results:
if exc is not None:
future.set_exception(exc)
else:
future.set(result)
if num_splits > 0:
_log.warn("Split batches complete. Number of splits: %s",
num_splits)
@staticmethod
def __split_batch(current_batch, remaining_batches):
"""
Splits batch in half and prepends it to the list of remaining
batches. Modifies remaining_batches in-place.
:param list[Message] current_batch: list of messages that's currently
being processed.
:param list[list[Message]] remaining_batches: list of batches
still to process.
"""
assert len(current_batch) > 1, "Batch too small to split"
# Split the batch.
split_point = len(current_batch) // 2
_log.debug("Split-point = %s", split_point)
first_half = current_batch[:split_point]
second_half = current_batch[split_point:]
if remaining_batches and not remaining_batches[0][0].needs_own_batch:
# Optimization: there's another batch already queued and
# it also contains batchable messages push the second
# half of this batch onto the front of that one.
_log.debug("Split batch and found a subsequent batch, "
"coalescing with that.")
next_batch = remaining_batches[0]
next_batch[:0] = second_half
else:
_log.debug("Split batch but cannot prepend to next batch, adding "
"both splits to start of queue.")
remaining_batches[:0] = [second_half]
remaining_batches[:0] = [first_half]
def _start_msg_batch(self, batch):
"""
Called before processing a batch of messages to give subclasses
a chance to filter the batch. Implementations must ensure that
every AsyncResult in the batch is correctly set. Usually, that
means combining them into one list.
It is usually easier to build up a batch of changes to make in the
@actor_message-decorated methods and then process them in
_finish_msg_batch().
Intended to be overridden. This implementation simply returns the
input batch.
:param list[Message] batch:
"""
return batch
def _finish_msg_batch(self, batch, results):
"""
Called after a batch of events have been processed from the queue
before results are set.
Intended to be overridden. This implementation does nothing.
Exceptions raised by this method are propagated to all messages in the
batch, overriding the existing results. It is recommended that the
implementation catches appropriate exceptions and maps them back
to the correct entry in results.
:param list[ResultOrExc] results: Pairs of (result, exception)
representing the result of each message-processing function.
Only one of the values is set. Updates to the list alter the
result send to any waiting listeners.
:param list[Message] batch: The input batch, always the same length as
results.
"""
pass
def _maybe_yield(self):
"""
With some probability, yields processing to another greenlet.
(Utility method to be called from the actor's greenlet during
long-running operations.)
"""
self._op_count += 1
if self._op_count >= self.max_ops_before_yield:
gevent.sleep()
self._op_count = 0
def __str__(self):
return self.__class__.__name__ + "<queue_len=%s,live=%s,msg=%s>" % (
self._event_queue.qsize(),
bool(self.greenlet),
self._current_msg
)
class OutputBatch(ActorBaseFT):
def __init__(self, name, chassis, config):
self._queue = None
super(OutputBatch, self).__init__(name, chassis, config)
self._push_glet = None
self._checkpoint_glet = None
def configure(self):
super(OutputBatch, self).configure()
self.queue_maxsize = int(self.config.get('queue_maxsize', 100000))
if self.queue_maxsize == 0:
self.queue_maxsize = None
self._queue = Queue(maxsize=self.queue_maxsize)
self.client_id = self.config.get('client_id', None)
self.client_secret = self.config.get('client_secret', None)
self.tenant_id = self.config.get('tenant_id', None)
self.action = self.config.get('action', 'Alert')
self.severity = self.config.get('severity', None)
self.side_config_path = self.config.get('side_config', None)
if self.side_config_path is None:
self.side_config_path = os.path.join(
os.environ['MM_CONFIG_DIR'], '%s_side_config.yml' % self.name)
self._load_side_config()
def _load_side_config(self):
try:
with open(self.side_config_path, 'r') as f:
sconfig = yaml.safe_load(f)
except Exception as e:
LOG.error('%s - Error loading side config: %s', self.name, str(e))
return
client_id = sconfig.get('client_id', None)
if client_id is not None:
self.client_id = client_id
LOG.info('{} - client_id set'.format(self.name))
client_secret = sconfig.get('client_secret', None)
if client_secret is not None:
self.client_secret = client_secret
LOG.info('{} - client_secret set'.format(self.name))
tenant_id = sconfig.get('tenant_id', None)
if tenant_id is not None:
self.tenant_id = tenant_id
LOG.info('{} - tenant_id set'.format(self.name))
action = sconfig.get('action', None)
if action is not None:
self.action = action
LOG.info('{} - action set'.format(self.action))
def connect(self, inputs, output):
output = False
super(OutputBatch, self).connect(inputs, output)
def _initialize_table(self, truncate=False):
self.table = Table(name=self.name, truncate=truncate)
def initialize(self):
self._initialize_table()
def rebuild(self):
self._initialize_table(truncate=(self.last_checkpoint is None))
def reset(self):
self._initialize_table(truncate=True)
def _get_auth_token(self):
if self.client_id is None:
LOG.error('{} - client_id not set'.format(self.name))
raise AuthConfigException('{} - client_id not set'.format(
self.name))
if self.client_secret is None:
LOG.error('{} - client_secret not set'.format(self.name))
raise AuthConfigException('{} - client_secret not set'.format(
self.name))
if self.tenant_id is None:
LOG.error('{} - tenant_id not set'.format(self.name))
raise AuthConfigException('{} - tenant_id not set'.format(
self.name))
context = adal.AuthenticationContext(
AUTHORITY_URL.format(self.tenant_id),
validate_authority=self.tenant_id != 'adfs',
api_version=None)
token = context.acquire_token_with_client_credentials(
RESOURCE, self.client_id, self.client_secret)
if token is None or 'accessToken' not in token:
LOG.error('{} - Invalid token or accessToken not available'.format(
self.name))
raise RuntimeError(
'{} - Invalid token or accessToken not available'.format(
self.name))
return token['accessToken']
def _push_indicators(self, token, indicators):
message = {'Indicators': list(indicators)}
LOG.debug(message)
result = requests.post(WD_ATP_TIINDICATORS_ENDPOINT,
headers={
'Content-Type': 'application/json',
'Authorization': 'Bearer {}'.format(token)
},
json=message)
LOG.debug(result.text)
result.raise_for_status()
# Check the status of the submitted indicators
# NOTE: if the indicator contains a range split by _encode_indicators, a partial submission might go through
# i.e. 192.168.0.1-192.168.0.3 can be split in 192.168.0.1, 192.168.0.2 and 192.168.0.3
# the first might go through, the second might return error
# This output node doesn't check for this condition (although the error counters are correctly updated)
result = result.json()
if not result or '@odata.context' not in result or result[
'@odata.context'] != 'https://api.securitycenter.windows.com/api/$metadata#Collection(microsoft.windowsDefenderATP.api.ImportIndicatorResult)':
raise WDATPResponseException('Unexpected response from WDATP API')
if 'value' not in result:
raise WDATPResponseException('Missing value from WDATP API result')
for v in result['value']:
if 'indicator' not in v or 'isFailed' not in v:
raise WDATPResponseException(
'Missing indicator values from WDATP response')
LOG.debug(
'{} - Got result for indicator {}: isFailed is {}'.format(
self.name, v['indicator'], v["isFailed"]))
if not v["isFailed"]:
# Success!
self.statistics['indicator.tx'] += 1
else:
failReason = v[
'failureReason'] if 'failureReason' in v else 'Unknown'
LOG.error('{}: error submitting indicator {}: {}'.format(
self.name, v['indicator'], failReason))
self.statistics['error.submit'] += 1
def _push_loop(self):
while True:
msg = self._queue.get()
artifacts = deque()
artifacts.append(msg)
try:
while len(artifacts) < 50:
artifacts.append(self._queue.get_nowait())
except Empty:
pass
while True:
retries = 0
try:
LOG.info('{} - Sending {} indicators'.format(
self.name, len(artifacts)))
token = self._get_auth_token()
LOG.debug('{} - token: {}'.format(self.name, token))
self._push_indicators(token=token, indicators=artifacts)
# Counter already incremented in push_indicators
# self.statistics['indicator.tx'] += len(artifacts)
break
except gevent.GreenletExit:
return
except HTTPError as e:
LOG.error('{} - error submitting indicators - {}'.format(
self.name, str(e)))
status_code = e.response.status_code
if status_code >= 400 and status_code < 500:
LOG.error('{}: error in request - {}'.format(
self.name, e.response.text))
self.statistics['error.invalid_request'] += 1
break
self.statistics['error.submit'] += 1
gevent.sleep(60)
except AuthConfigException as e:
LOG.exception(
'{} - Error submitting indicators - {}'.format(
self.name, str(e)))
self.statistics['error.submit'] += 1
gevent.sleep(60.0)
except WDATPResponseException as e:
LOG.exception(
'{} - error submitting indicators - {}'.format(
self.name, str(e)))
self.statistics['error.submit'] += 1
break
except Exception as e:
LOG.exception(
'{} - error submitting indicators - {}'.format(
self.name, str(e)))
self.statistics['error.submit'] += 1
retries += 1
if retries > 5:
break
gevent.sleep(120.0)
gevent.sleep(0.1)
def _encode_indicator(self, indicator, value, expired=False):
type_ = MM_2_WDATP_TYPE.get(value['type'], None)
if type_ is None:
self.statistics['error.unhandled_type'] += 1
raise RuntimeError('{} - Unhandled {}'.format(self.name, type_))
if value['type'] == 'IPv4' and '-' in indicator:
a1, a2 = indicator.split('-', 1)
r = netaddr.IPRange(a1, a2)
indicators = [str(i) for i in r]
else:
indicators = [indicator]
description = '{} indicator from {}'.format(
type_, ', '.join(value['sources']))
title = 'MineMeld - {}'.format(indicator)
creation = datetime.utcnow()
creation = creation.isoformat() + 'Z'
expiration = datetime.utcnow() + timedelta(days=365)
if expired:
expiration = datetime.fromtimestamp(0)
expiration = expiration.isoformat(
) + 'Z' # expiration is always in UTC
result = []
for i in indicators:
d = dict(indicatorValue=i,
indicatorType=type_,
title=title,
description=description,
creationTimeDateTimeUtc=creation,
expirationTime=expiration,
action=self.action)
if self.severity is not None:
d['severity'] = self.severity
result.append(d)
return result
def _checkpoint_check(self, source=None, value=None):
t0 = time.time()
while ((time.time() - t0) < 30) and self._queue.qsize() != 0:
gevent.sleep(0.5)
self._push_glet.kill()
LOG.info('{} - checkpoint with {} elements in the queue'.format(
self.name, self._queue.qsize()))
super(OutputBatch, self).checkpoint(source=source, value=value)
@_counting('update.processed')
def filtered_update(self, source=None, indicator=None, value=None):
try:
for i in self._encode_indicator(indicator, value, expired=False):
self._queue.put(i, block=True, timeout=0.001)
except Full:
self.statistics['error.queue_full'] += 1
@_counting('withdraw.processed')
def filtered_withdraw(self, source=None, indicator=None, value=None):
if value is None:
self.statistics['error.no_value'] += 1
return
try:
for i in self._encode_indicator(indicator, value, expired=True):
self._queue.put(i, block=True, timeout=0.001)
except Full:
self.statistics['error.queue_full'] += 1
@_counting('checkpoint.rx')
def checkpoint(self, source=None, value=None):
self.state = ft_states.CHECKPOINT
self._checkpoint_glet = gevent.spawn(self._checkpoint_check, source,
value)
def length(self, source=None):
return self._queue.qsize()
def start(self):
super(OutputBatch, self).start()
self._push_glet = gevent.spawn(self._push_loop)
def stop(self):
super(OutputBatch, self).stop()
if self._push_glet is not None:
self._push_glet.kill()
if self._checkpoint_glet is not None:
self._checkpoint_glet.kill()
self.table.close()
def hup(self, source=None):
LOG.info('%s - hup received, reload side config', self.name)
self._load_side_config()
@staticmethod
def gc(name, config=None):
ActorBaseFT.gc(name, config=config)
shutil.rmtree(name, ignore_errors=True)
class Connection(object):
in_buffer_size = 4096
out_buffer_size = 4096
cql_version = None
protocol_version = 2
keyspace = None
compression = True
compressor = None
decompressor = None
ssl_options = None
last_error = None
in_flight = 0
is_defunct = False
is_closed = False
lock = None
is_control_connection = False
def __init__(self, host='127.0.0.1', port=9042, authenticator=None,
ssl_options=None, sockopts=None, compression=True,
cql_version=None, protocol_version=2, is_control_connection=False):
self.host = host
self.port = port
self.authenticator = authenticator
self.ssl_options = ssl_options
self.sockopts = sockopts
self.compression = compression
self.cql_version = cql_version
self.protocol_version = protocol_version
self.is_control_connection = is_control_connection
self._push_watchers = defaultdict(set)
self._id_queue = Queue(MAX_STREAM_PER_CONNECTION)
for i in range(MAX_STREAM_PER_CONNECTION):
self._id_queue.put_nowait(i)
self.lock = RLock()
@classmethod
def initialize_reactor(self):
"""
Called once by Cluster.connect(). This should be used by implementations
to set up any resources that will be shared across connections.
"""
pass
def close(self):
raise NotImplementedError()
def defunct(self, exc):
with self.lock:
if self.is_defunct or self.is_closed:
return
self.is_defunct = True
log.debug("Defuncting connection (%s) to %s:",
id(self), self.host, exc_info=exc)
self.last_error = exc
self.close()
self.error_all_callbacks(exc)
self.connected_event.set()
return exc
def error_all_callbacks(self, exc):
with self.lock:
callbacks = self._callbacks
self._callbacks = {}
new_exc = ConnectionShutdown(str(exc))
for cb in callbacks.values():
try:
cb(new_exc)
except Exception:
log.warning("Ignoring unhandled exception while erroring callbacks for a "
"failed connection (%s) to host %s:",
id(self), self.host, exc_info=True)
def handle_pushed(self, response):
log.debug("Message pushed from server: %r", response)
for cb in self._push_watchers.get(response.event_type, []):
try:
cb(response.event_args)
except Exception:
log.exception("Pushed event handler errored, ignoring:")
def send_msg(self, msg, cb, wait_for_id=False):
if self.is_defunct:
raise ConnectionShutdown("Connection to %s is defunct" % self.host)
elif self.is_closed:
raise ConnectionShutdown("Connection to %s is closed" % self.host)
if not wait_for_id:
try:
request_id = self._id_queue.get_nowait()
except Empty:
raise ConnectionBusy(
"Connection to %s is at the max number of requests" % self.host)
else:
request_id = self._id_queue.get()
self._callbacks[request_id] = cb
self.push(msg.to_binary(request_id, self.protocol_version, compression=self.compressor))
return request_id
def wait_for_response(self, msg, timeout=None):
return self.wait_for_responses(msg, timeout=timeout)[0]
def wait_for_responses(self, *msgs, **kwargs):
if self.is_closed or self.is_defunct:
raise ConnectionShutdown("Connection %s is already closed" % (self, ))
timeout = kwargs.get('timeout')
waiter = ResponseWaiter(self, len(msgs))
# busy wait for sufficient space on the connection
messages_sent = 0
while True:
needed = len(msgs) - messages_sent
with self.lock:
available = min(needed, MAX_STREAM_PER_CONNECTION - self.in_flight)
self.in_flight += available
for i in range(messages_sent, messages_sent + available):
self.send_msg(msgs[i], partial(waiter.got_response, index=i), wait_for_id=True)
messages_sent += available
if messages_sent == len(msgs):
break
else:
if timeout is not None:
timeout -= 0.01
if timeout <= 0.0:
raise OperationTimedOut()
time.sleep(0.01)
try:
return waiter.deliver(timeout)
except OperationTimedOut:
raise
except Exception as exc:
self.defunct(exc)
raise
def register_watcher(self, event_type, callback):
raise NotImplementedError()
def register_watchers(self, type_callback_dict):
raise NotImplementedError()
def control_conn_disposed(self):
self.is_control_connection = False
self._push_watchers = {}
@defunct_on_error
def process_msg(self, msg, body_len):
version, flags, stream_id, opcode = header_unpack(msg[:4])
if stream_id < 0:
callback = None
else:
callback = self._callbacks.pop(stream_id, None)
self._id_queue.put_nowait(stream_id)
body = None
try:
# check that the protocol version is supported
given_version = version & PROTOCOL_VERSION_MASK
if given_version != self.protocol_version:
msg = "Server protocol version (%d) does not match the specified driver protocol version (%d). " +\
"Consider setting Cluster.protocol_version to %d."
raise ProtocolError(msg % (given_version, self.protocol_version, given_version))
# check that the header direction is correct
if version & HEADER_DIRECTION_MASK != HEADER_DIRECTION_TO_CLIENT:
raise ProtocolError(
"Header direction in response is incorrect; opcode %04x, stream id %r"
% (opcode, stream_id))
if body_len > 0:
body = msg[8:]
elif body_len == 0:
body = six.binary_type()
else:
raise ProtocolError("Got negative body length: %r" % body_len)
response = decode_response(stream_id, flags, opcode, body, self.decompressor)
except Exception as exc:
log.exception("Error decoding response from Cassandra. "
"opcode: %04x; message contents: %r", opcode, msg)
if callback is not None:
callback(exc)
self.defunct(exc)
return
try:
if stream_id < 0:
self.handle_pushed(response)
elif callback is not None:
callback(response)
except Exception:
log.exception("Callback handler errored, ignoring:")
@defunct_on_error
def _send_options_message(self):
if self.cql_version is None and (not self.compression or not locally_supported_compressions):
log.debug("Not sending options message for new connection(%s) to %s "
"because compression is disabled and a cql version was not "
"specified", id(self), self.host)
self._compressor = None
self.cql_version = DEFAULT_CQL_VERSION
self._send_startup_message()
else:
log.debug("Sending initial options message for new connection (%s) to %s", id(self), self.host)
self.send_msg(OptionsMessage(), self._handle_options_response)
@defunct_on_error
def _handle_options_response(self, options_response):
if self.is_defunct:
return
if not isinstance(options_response, SupportedMessage):
if isinstance(options_response, ConnectionException):
raise options_response
else:
log.error("Did not get expected SupportedMessage response; " \
"instead, got: %s", options_response)
raise ConnectionException("Did not get expected SupportedMessage " \
"response; instead, got: %s" \
% (options_response,))
log.debug("Received options response on new connection (%s) from %s",
id(self), self.host)
supported_cql_versions = options_response.cql_versions
remote_supported_compressions = options_response.options['COMPRESSION']
if self.cql_version:
if self.cql_version not in supported_cql_versions:
raise ProtocolError(
"cql_version %r is not supported by remote (w/ native "
"protocol). Supported versions: %r"
% (self.cql_version, supported_cql_versions))
else:
self.cql_version = supported_cql_versions[0]
self._compressor = None
compression_type = None
if self.compression:
overlap = (set(locally_supported_compressions.keys()) &
set(remote_supported_compressions))
if len(overlap) == 0:
log.debug("No available compression types supported on both ends."
" locally supported: %r. remotely supported: %r",
locally_supported_compressions.keys(),
remote_supported_compressions)
else:
compression_type = None
if isinstance(self.compression, six.string_types):
# the user picked a specific compression type ('snappy' or 'lz4')
if self.compression not in remote_supported_compressions:
raise ProtocolError(
"The requested compression type (%s) is not supported by the Cassandra server at %s"
% (self.compression, self.host))
compression_type = self.compression
else:
# our locally supported compressions are ordered to prefer
# lz4, if available
for k in locally_supported_compressions.keys():
if k in overlap:
compression_type = k
break
# set the decompressor here, but set the compressor only after
# a successful Ready message
self._compressor, self.decompressor = \
locally_supported_compressions[compression_type]
self._send_startup_message(compression_type)
@defunct_on_error
def _send_startup_message(self, compression=None):
opts = {}
if compression:
opts['COMPRESSION'] = compression
sm = StartupMessage(cqlversion=self.cql_version, options=opts)
self.send_msg(sm, cb=self._handle_startup_response)
@defunct_on_error
def _handle_startup_response(self, startup_response, did_authenticate=False):
if self.is_defunct:
return
if isinstance(startup_response, ReadyMessage):
log.debug("Got ReadyMessage on new connection (%s) from %s", id(self), self.host)
if self._compressor:
self.compressor = self._compressor
self.connected_event.set()
elif isinstance(startup_response, AuthenticateMessage):
log.debug("Got AuthenticateMessage on new connection (%s) from %s: %s",
id(self), self.host, startup_response.authenticator)
if self.authenticator is None:
raise AuthenticationFailed('Remote end requires authentication.')
self.authenticator_class = startup_response.authenticator
if isinstance(self.authenticator, dict):
log.debug("Sending credentials-based auth response on %s", self)
cm = CredentialsMessage(creds=self.authenticator)
callback = partial(self._handle_startup_response, did_authenticate=True)
self.send_msg(cm, cb=callback)
else:
log.debug("Sending SASL-based auth response on %s", self)
initial_response = self.authenticator.initial_response()
initial_response = "" if initial_response is None else initial_response.encode('utf-8')
self.send_msg(AuthResponseMessage(initial_response), self._handle_auth_response)
elif isinstance(startup_response, ErrorMessage):
log.debug("Received ErrorMessage on new connection (%s) from %s: %s",
id(self), self.host, startup_response.summary_msg())
if did_authenticate:
raise AuthenticationFailed(
"Failed to authenticate to %s: %s" %
(self.host, startup_response.summary_msg()))
else:
raise ConnectionException(
"Failed to initialize new connection to %s: %s"
% (self.host, startup_response.summary_msg()))
elif isinstance(startup_response, ConnectionShutdown):
log.debug("Connection to %s was closed during the startup handshake", (self.host))
raise startup_response
else:
msg = "Unexpected response during Connection setup: %r"
log.error(msg, startup_response)
raise ProtocolError(msg % (startup_response,))
@defunct_on_error
def _handle_auth_response(self, auth_response):
if self.is_defunct:
return
if isinstance(auth_response, AuthSuccessMessage):
log.debug("Connection %s successfully authenticated", self)
self.authenticator.on_authentication_success(auth_response.token)
if self._compressor:
self.compressor = self._compressor
self.connected_event.set()
elif isinstance(auth_response, AuthChallengeMessage):
response = self.authenticator.evaluate_challenge(auth_response.challenge)
msg = AuthResponseMessage("" if response is None else response)
log.debug("Responding to auth challenge on %s", self)
self.send_msg(msg, self._handle_auth_response)
elif isinstance(auth_response, ErrorMessage):
log.debug("Received ErrorMessage on new connection (%s) from %s: %s",
id(self), self.host, auth_response.summary_msg())
raise AuthenticationFailed(
"Failed to authenticate to %s: %s" %
(self.host, auth_response.summary_msg()))
elif isinstance(auth_response, ConnectionShutdown):
log.debug("Connection to %s was closed during the authentication process", self.host)
raise auth_response
else:
msg = "Unexpected response during Connection authentication to %s: %r"
log.error(msg, self.host, auth_response)
raise ProtocolError(msg % (self.host, auth_response))
def set_keyspace_blocking(self, keyspace):
if not keyspace or keyspace == self.keyspace:
return
query = QueryMessage(query='USE "%s"' % (keyspace,),
consistency_level=ConsistencyLevel.ONE)
try:
result = self.wait_for_response(query)
except InvalidRequestException as ire:
# the keyspace probably doesn't exist
raise ire.to_exception()
except Exception as exc:
conn_exc = ConnectionException(
"Problem while setting keyspace: %r" % (exc,), self.host)
self.defunct(conn_exc)
raise conn_exc
if isinstance(result, ResultMessage):
self.keyspace = keyspace
else:
conn_exc = ConnectionException(
"Problem while setting keyspace: %r" % (result,), self.host)
self.defunct(conn_exc)
raise conn_exc
def set_keyspace_async(self, keyspace, callback):
"""
Use this in order to avoid deadlocking the event loop thread.
When the operation completes, `callback` will be called with
two arguments: this connection and an Exception if an error
occurred, otherwise :const:`None`.
"""
if not keyspace or keyspace == self.keyspace:
callback(self, None)
return
query = QueryMessage(query='USE "%s"' % (keyspace,),
consistency_level=ConsistencyLevel.ONE)
def process_result(result):
if isinstance(result, ResultMessage):
self.keyspace = keyspace
callback(self, None)
elif isinstance(result, InvalidRequestException):
callback(self, result.to_exception())
else:
callback(self, self.defunct(ConnectionException(
"Problem while setting keyspace: %r" % (result,), self.host)))
self.send_msg(query, process_result, wait_for_id=True)
def __str__(self):
status = ""
if self.is_defunct:
status = " (defunct)"
elif self.is_closed:
status = " (closed)"
return "<%s(%r) %s:%d%s>" % (self.__class__.__name__, id(self), self.host, self.port, status)
__repr__ = __str__
class Output(ActorBaseFT):
def __init__(self, name, chassis, config):
self._queue = None
super(Output, self).__init__(name, chassis, config)
self._push_glet = None
self._checkpoint_glet = None
self.api_client_id = str(uuid.uuid4())
self.sequence_number = 0
def configure(self):
super(Output, self).configure()
self.queue_maxsize = int(self.config.get('queue_maxsize', 100000))
if self.queue_maxsize == 0:
self.queue_maxsize = None
self._queue = Queue(maxsize=self.queue_maxsize)
self.client_id = self.config.get('client_id', None)
self.client_secret = self.config.get('client_secret', None)
self.tenant_id = self.config.get('tenant_id', None)
self.sender_id = self.config.get('sender_id', 'minemeld')
self.side_config_path = self.config.get('side_config', None)
if self.side_config_path is None:
self.side_config_path = os.path.join(
os.environ['MM_CONFIG_DIR'], '%s_side_config.yml' % self.name)
self._load_side_config()
def _load_side_config(self):
try:
with open(self.side_config_path, 'r') as f:
sconfig = yaml.safe_load(f)
except Exception as e:
LOG.error('%s - Error loading side config: %s', self.name, str(e))
return
client_id = sconfig.get('client_id', None)
if client_id is not None:
self.client_id = client_id
LOG.info('{} - client_id set'.format(self.name))
client_secret = sconfig.get('client_secret', None)
if client_secret is not None:
self.client_secret = client_secret
LOG.info('{} - client_secret set'.format(self.name))
tenant_id = sconfig.get('tenant_id', None)
if tenant_id is not None:
self.tenant_id = tenant_id
LOG.info('{} - tenant_id set'.format(self.name))
def _saved_state_restore(self, saved_state):
super(Output, self)._saved_state_restore(saved_state)
self.api_client_id = saved_state.get('api_client_id', None)
self.sequence_number = saved_state.get('sequence_number', None)
LOG.info(
'{} - saved state: api_client_id: {} sequence_number: {}'.format(
self.name, self.api_client_id, self.sequence_number))
def _saved_state_create(self):
sstate = super(Output, self)._saved_state_create()
sstate['api_client_id'] = self.api_client_id
sstate['sequence_number'] = self.sequence_number
return sstate
def _saved_state_reset(self):
super(Output, self)._saved_state_reset()
self.api_client_id = str(uuid.uuid4())
self.sequence_number = 0
def connect(self, inputs, output):
output = False
super(Output, self).connect(inputs, output)
def initialize(self):
pass
def rebuild(self):
pass
def reset(self):
pass
def _get_auth_token(self):
if self.client_id is None:
LOG.error('{} - client_id not set'.format(self.name))
raise AuthConfigException('{} - client_id not set'.format(
self.name))
if self.client_secret is None:
LOG.error('{} - client_secret not set'.format(self.name))
raise AuthConfigException('{} - client_secret not set'.format(
self.name))
if self.tenant_id is None:
LOG.error('{} - tenant_id not set'.format(self.name))
raise AuthConfigException('{} - tenant_id not set'.format(
self.name))
context = adal.AuthenticationContext(
AUTHORITY_URL.format(self.tenant_id),
validate_authority=self.tenant_id != 'adfs',
api_version=None)
token = context.acquire_token_with_client_credentials(
RESOURCE, self.client_id, self.client_secret)
if token is None or 'accessToken' not in token:
LOG.error('{} - Invalid token or accessToken not available'.format(
self.name))
raise RuntimeError(
'{} - Invalid token or accessToken not available'.format(
self.name))
return token['accessToken']
def _get_endpoint_orgid(self, token):
# this should look like
# {
# u'AadTenantId': u'bb19bb5c-0e8d-4a73-bd6b-d015b298ecd7',
# u'ServiceUri': u'https://partnerstifrontend-eus-prd.trafficmanager.net/threatintel/indicators',
# u'ServiceType': 1,
# u'WdAtpOrgId': u'55c01df7-a1eb-4eae-ae3b-a9b423d07d72'
# }
result = requests.get(ENDPOINT_URL,
headers={
'Authorization': 'Bearer {}'.format(token),
'Content-Type': 'application/json'
})
result.raise_for_status()
result = result.json()
LOG.debug('{} - endpoints: {}'.format(self.name, result))
if result.get('AadTenantId', None) != self.tenant_id:
raise AuthConfigException(
'{} - Endpoint response AadTenantId differs from tenant_id: {}'
.format(self.name, result))
endpoint = result.get('ServiceUri', None)
if endpoint is None:
raise AuthConfigException(
'{} - Endpoint response missing ServiceUri field: {}'.format(
self.name, result))
org_id = result.get('WdAtpOrgId', None)
if endpoint is None:
raise AuthConfigException(
'{} - Endpoint response missing WdAtpOrgId field: {}'.format(
self.name, result))
return endpoint, org_id
def _push_indicators(self, token, endpoint, org_id, indicators):
# DEPRECATED
# message = {
# 'Id': self.api_client_id,
# 'SequenceNumber': self.sequence_number,
# 'SenderId': self.sender_id,
# 'Indicators': list(indicators),
# 'WdAtpOrgId': org_id
# }
# LOG.debug(message)
# result = requests.post(
# endpoint,
# headers={
# 'Content-Type': 'application/json',
# 'Authorization': 'Bearer {}'.format(token)
# },
# json=message
# )
# LOG.debug(result.text)
# result.raise_for_status()
raise WDATPResponseException(
'This output node is deprecated: please switch to OutputBatch')
def _push_loop(self):
while True:
msg = self._queue.get()
artifacts = deque()
artifacts.append(msg)
try:
while len(artifacts) < 511:
artifacts.append(self._queue.get_nowait())
except Empty:
pass
while True:
result = None
try:
LOG.info('{} - Sending {}:{}'.format(
self.name, self.api_client_id, self.sequence_number))
# DEPRECATED - no need to get the token
# token = self._get_auth_token()
token = 'DEPRECATED'
LOG.debug('{} - token: {}'.format(self.name, token))
# DEPRECATED
#endpoint, org_id = self._get_endpoint_orgid(token)
#LOG.debug('{} - endpoint: {} WdAtpOrgId: {}'.format(self.name, endpoint, org_id))
# self._push_indicators(
# token=token,
# endpoint=endpoint,
# org_id=org_id,
# indicators=artifacts
# )
self._push_indicators(None, None, None, None)
self.sequence_number += 1
self.statistics['indicator.tx'] += len(artifacts)
break
except gevent.GreenletExit:
return
except RequestException as e:
LOG.error('{} - error submitting indicators - {}'.format(
self.name, str(e)))
if result is not None and result.status_code >= 400 and result.status_code < 500:
LOG.error('{}: error in request - {}'.format(
self.name, result.text))
self.statistics['error.invalid_request'] += 1
break
self.statistics['error.submit'] += 1
gevent.sleep(60)
except AuthConfigException as e:
LOG.exception(
'{} - Error submitting indicators - {}'.format(
self.name, str(e)))
self.statistics['error.submit'] += 1
gevent.sleep(60.0)
except WDATPResponseException as e:
LOG.exception(
'{} - error submitting indicators - {}'.format(
self.name, str(e)))
self.statistics['error.submit'] += 1
break
except Exception as e:
LOG.exception(
'{} - error submitting indicators - {}'.format(
self.name, str(e)))
self.statistics['error.submit'] += 1
gevent.sleep(120.0)
gevent.sleep(0.1)
def _encode_indicator(self, indicator, value, expired=False):
type_ = value['type']
if type_ not in ['URL', 'domain', 'md5', 'sha256', 'IPv4']:
self.statistics['error.unhandled_type'] += 1
raise RuntimeError('{} - Unhandled {}'.format(self.name, type_))
description = '{} indicator from {}'.format(
type_, ', '.join(value['sources']))
external_id = '{}:{}'.format(type_, indicator)
expiration = datetime.utcnow() + timedelta(days=365)
if expired:
expiration = datetime.fromtimestamp(0)
expiration = expiration.isoformat()
indicators = []
if type_ == 'IPv4' and '-' in indicator:
# October 2020
# as MSFT removed the support for CIDRs in the API
# we must translate the ranges in individual IPv4 indicators
a1, a2 = indicator.split('-', 1)
r = netaddr.IPRange(a1, a2)
indicators = [str(i) for i in r]
else:
indicators = [indicator]
result = []
for i in indicators:
r = {
'Description': description,
'Confidence': value['confidence'],
'ExternalId': external_id,
'IndicatorExpirationDateTime': expiration
}
if type_ == 'URL':
r['Url'] = indicator
elif type_ == 'domain':
r['DNSDomainName'] = indicator
elif type_ == 'md5':
r['FileMD5'] = indicator
elif type_ == 'sha256':
r['FileSha256'] = indicator
elif type_ == 'IPv4':
r['NetworkDestinationIPv4'] = str(indicator)
else:
# Unsupported indicator type, should never reach this code
continue
LOG.debug('{!r} - add indicator {!r} to queue'.format(
self.name, r))
result.append(r)
return result
def _checkpoint_check(self, source=None, value=None):
t0 = time.time()
while ((time.time() - t0) < 30) and self._queue.qsize() != 0:
gevent.sleep(0.5)
self._push_glet.kill()
LOG.info('{} - checkpoint with {} elements in the queue'.format(
self.name, self._queue.qsize()))
super(Output, self).checkpoint(source=source, value=value)
@_counting('update.processed')
def filtered_update(self, source=None, indicator=None, value=None):
try:
for i in self._encode_indicator(indicator, value, expired=False):
self._queue.put(i, block=True, timeout=0.001)
except Full:
self.statistics['error.queue_full'] += 1
@_counting('withdraw.processed')
def filtered_withdraw(self, source=None, indicator=None, value=None):
if value is None:
self.statistics['error.no_value'] += 1
return
try:
for i in self._encode_indicator(indicator, value, expired=True):
self._queue.put(i, block=True, timeout=0.001)
except Full:
self.statistics['error.queue_full'] += 1
@_counting('checkpoint.rx')
def checkpoint(self, source=None, value=None):
self.state = ft_states.CHECKPOINT
self._checkpoint_glet = gevent.spawn(self._checkpoint_check, source,
value)
def mgmtbus_status(self):
result = super(ActorBaseFT, self).mgmtbus_status()
result['sub_state'] = 'ERROR'
result['sub_state_message'] = 'This node is deprecated'
return result
def length(self, source=None):
return self._queue.qsize()
def start(self):
super(Output, self).start()
self._push_glet = gevent.spawn(self._push_loop)
def stop(self):
super(Output, self).stop()
if self._push_glet is not None:
self._push_glet.kill()
if self._checkpoint_glet is not None:
self._checkpoint_glet.kill()
def hup(self, source=None):
LOG.info('%s - hup received, reload side config', self.name)
self._load_side_config()
@staticmethod
def gc(name, config=None):
ActorBaseFT.gc(name, config=config)
shutil.rmtree(name, ignore_errors=True)
