def test_default_anonymous_no_rate_limit( client, client_ip, switch_on, mocker, url): def fake_switch(name, default=True): if name == SWITCH_ENABLE_RATE_LIMITER: return switch_on return default mocker.patch.object(switch, 'is_switched_on', fake_switch) if not switch_on: mocker.patch.object(settings, 'RATE_LIMITER_SETTINGS', { '__default__': { 'rate': 1, 'capacity': 3, } }) def worker(queue): response = client.get(url) if response.status_code == 429: queue.put(True) greenlets = [] queue = Queue() for _ in range(3): greenlets.append(gevent.spawn(worker, queue)) gevent.joinall(greenlets) with raises(Empty): queue.get_nowait()
def test_logged_concurrent_limit_with_redis_error( client, client_ip, test_user, test_token, mocker, error_method, cause_limit): mocker.patch.object(settings, 'CONCURRENT_LIMITER_SETTINGS', { test_user.username: { 'ttl': 100, 'capacity': 1, } }) mocker.patch.object(redis_client, error_method, side_effect=Exception) def worker(queue): response = client.get( '/api/busy_with_login', headers={ 'Authorization': test_token, }) if response.status_code == 429: queue.put(429) greenlets = [] queue = Queue() for _ in range(3): greenlets.append(gevent.spawn(worker, queue)) gevent.joinall(greenlets) if cause_limit: assert queue.get_nowait() == 429 else: with raises(Empty): queue.get_nowait()
def test_default_logged_no_concurrent_limit( client, client_ip, test_token, switch_on, mocker): def fake_switch(name, default=True): if name == SWITCH_ENABLE_CONCURRENT_LIMITER: return switch_on return default mocker.patch.object(switch, 'is_switched_on', fake_switch) if not switch_on: mocker.patch.object(settings, 'CONCURRENT_LIMITER_SETTINGS', { '__default__': { 'ttl': 100, 'capacity': 1, } }) def worker(queue): response = client.get( '/api/busy_with_login', headers={ 'Authorization': test_token, }) if response.status_code == 429: queue.put(429) greenlets = [] queue = Queue() for _ in range(3): greenlets.append(gevent.spawn(worker, queue)) gevent.joinall(greenlets) with raises(Empty): queue.get_nowait()
def test_logged_no_rate_limit_because_redis_error( client, client_ip, test_user, test_token, mocker, error_method): mocker.patch.object(settings, 'RATE_LIMITER_SETTINGS', { test_user.username: { 'rate': 1, 'capacity': 3, } }) mocker.patch.object(redis_client, error_method, side_effect=Exception) def worker(queue): response = client.get( '/api/need_login', headers={ 'Authorization': test_token, }) if response.status_code == 429: queue.put(429) greenlets = [] queue = Queue() for _ in range(3): greenlets.append(gevent.spawn(worker, queue)) gevent.joinall(greenlets) with raises(Empty): queue.get_nowait()
def test_logged_no_concurrent_limit_because_remain_count( client, client_ip, test_user, test_token, mocker): mocker.patch.object(settings, 'CONCURRENT_LIMITER_SETTINGS', { test_user.username: { 'ttl': 100, 'capacity': 100, } }) def worker(queue): response = client.get( '/api/busy_with_login', headers={ 'Authorization': test_token, }) if response.status_code == 429: queue.put(429) greenlets = [] queue = Queue() for _ in range(3): greenlets.append(gevent.spawn(worker, queue)) gevent.joinall(greenlets) with raises(Empty): queue.get_nowait()
def test_logged_with_rate_limit( client, client_ip, mocker, test_user, test_token, configs, use_username): if '127.0.0.1' in configs: configs[client_ip] = configs.pop('127.0.0.1') if use_username: configs.update({ test_user.username: { 'rate': 1, 'capacity': 3, } }) mocker.patch.object(settings, 'RATE_LIMITER_SETTINGS', configs) def worker(queue): response = client.get('/api/need_login', headers={ 'Authorization': test_token, }) if response.status_code == 429: queue.put(429) greenlets = [] queue = Queue() for _ in range(10): greenlets.append(gevent.spawn(worker, queue)) gevent.joinall(greenlets) assert queue.get_nowait() == 429
class LocalRPCServer(Greenlet): def __init__(self, server_queue): Greenlet.__init__(self) self.server_queue = server_queue self.recv_queue = Queue() self.handle_stoping = False self.recv_stop_evt = Event() def write(self, callInfo): self.recv_queue.put(callInfo) # 对RPC请求队列中的请求进行处理 def recv_call_handle(self): while True: if self.recv_queue.empty() & self.handle_stoping: self.recv_stop_evt.set() return if not self.recv_queue.empty(): callInfo = self.recv_queue.get_nowait() callInfo.agent = self self.server_queue.put(callInfo) gevent.sleep(0) def Callback(self, callInfo): reply_to = callInfo.props["reply_to"] reply_to.put(callInfo.Result) def _run(self): try: self.recv_call_handle() except Exception, e: print e
class EventLoop(): def __init__(self, strategy): self.strategy = strategy self.event_queue = Queue() def proc_strategy(self): global DELAY gevent.sleep(MAX_DELAY) status = self.strategy.run() if status == StrategyResult.SUCCESS: DELAY = max(MIN_DELAY, DELAY / 2) elif status == StrategyResult.FAILURE: self.event_queue.put(EQType.STOP_EXECUTION) elif status == StrategyResult.PASSED: DELAY = min(MAX_DELAY, DELAY * 2) def run(self): pool = Pool(config.max_concurrency) while True: pool.spawn(self.proc_strategy) try: next_op = self.event_queue.get_nowait() if next_op == EQType.STOP_EXECUTION: pool.join() break except gevent.queue.Empty: pass
class Crawler(object): def __init__(self, processor): self.processor = processor self.pool = Pool(self.processor.concurrency) self.base_host = urlsplit(self.processor.start_url).hostname self.urls = Queue() self.urls.put(self.processor.start_url) self.visited_urls = set() self.visited_urls_lock = RLock() self.pages_count = 0 def start(self): while True: if self.pages_count >= self.processor.max_pages: self.urls = Queue() break try: url = self.urls.get_nowait() self.pool.wait_available() spider = Spider(self, self.processor, url) self.pool.start(spider) self.pages_count += 1 except Empty: break self.pool.join() if not self.urls.empty(): self.start()
def test_login_with_concurrent_limit( client, client_ip, mocker, test_user, test_token, configs, use_username): if '127.0.0.1' in configs: configs[client_ip] = configs.pop('127.0.0.1') if use_username: configs.update({ test_user.username: { 'ttl': 100, 'capacity': 1, } }) mocker.patch.object(settings, 'CONCURRENT_LIMITER_SETTINGS', configs) def worker(queue): response = client.get('/api/busy_with_login', headers={ 'Authorization': test_token, }) if response.status_code != 200: queue.put(response.status_code) greenlets = [] queue = Queue() for _ in range(3): greenlets.append(gevent.spawn(worker, queue)) gevent.joinall(greenlets) assert queue.get_nowait() == 429
def _run(self): utils.log("[%s] parsing site %s" % (self, self.base)) queue = Queue() pool = Pool(64) seed = 'http://www.nytimes.com/best-sellers-books/' pool.spawn(self._parseResultsPage, pool, queue, seed, 'current', True) while True: items = [] while not queue.empty(): item = queue.get_nowait() items.append(item) if 0 == len(items) and 0 == len(pool): break for item in items: pool.spawn(item[0], pool, queue, item[1], item[2], item[3]) time.sleep(0.01) pool.join() self._output.put(StopIteration)
def _run(self): utils.log("[%s] parsing site %s" % (self, self.base)) queue = Queue() pool = Pool(64) seed = 'http://community.seattletimes.nwsource.com/entertainment/i_results.php?search=venue&type=Restaurant&page=1' pool.spawn(self._parseResultsPage, pool, queue, seed, '1', True) while True: items = [] while not queue.empty(): item = queue.get_nowait() items.append(item) if 0 == len(items) and 0 == len(pool): break for item in items: pool.spawn(item[0], pool, queue, item[1], item[2], item[3]) time.sleep(0.01) pool.join() self._output.put(StopIteration)
def _run(self): utils.log("[%s] parsing site %s" % (self, self.base)) queue = Queue() pool = Pool(32) seed = 'http://www.amazon.com/best-sellers-books-Amazon/zgbs/books/' parsed = set() queue.put_nowait((seed, 'seed', 0)) while True: items = [] while not queue.empty(): item = queue.get_nowait() if item[0] not in parsed: items.append(item) parsed.add(item[0]) if 0 == len(items) and 0 == len(pool): break for item in items: pool.spawn(self._parseResultsPage, queue, item[0], item[1], item[2]) time.sleep(0.01) pool.join() self._output.put(StopIteration)
def _run(self): utils.log("[%s] parsing site %s" % (self, self.base)) queue = Queue() pool = Pool(16) seed = 'http://www.awardannals.com/skin/menubar81.html' pool.spawn(self._parseIndexPage, pool, queue, seed, 'index') while True: items = [] while not queue.empty(): item = queue.get_nowait() items.append(item) if 0 == len(items) and 0 == len(pool): break for item in items: pool.spawn(self._parseResultsPage, pool, queue, item[0], item[1], False) time.sleep(0.01) pool.join() self._output.put(StopIteration)
class __AudioNode(gevent.Greenlet): RATE = 44100 CHUNK = 512 PORT = 20000 def __init__(self, is_log=True): gevent.Greenlet.__init__(self) self.is_log = is_log self.command = Queue() def is_quit(self): try: cmd = self.command.get_nowait() return cmd == "q" except Empty: return False def stop(self, msg=""): self.command.put("q") if self.is_log: print "%s - stopping %s" % (self.__class__.__name__, msg) def _run(self): if self.is_log: print "starting: %s" % self.__class__.__name__ try: self.engine() finally: self.pa.close() self.sock.close()
class BaseMailSyncMonitor(Greenlet): def __init__(self, account_id, email_address, provider, status_cb, heartbeat=1): self.inbox = Queue() # how often to check inbox, in seconds self.heartbeat = heartbeat self.log = configure_mailsync_logging(account_id) self.account_id = account_id self.email_address = email_address self.provider = provider # Stuff that might be updated later and we want to keep a shared # reference on child greenlets. if hasattr(self, 'shared_state'): self.shared_state['status_cb'] = status_cb else: self.shared_state = dict(status_cb=status_cb) Greenlet.__init__(self) def _run(self): return log_uncaught_errors(self._run_impl, self.log)() def _run_impl(self): sync = Greenlet.spawn(log_uncaught_errors(self.sync, self.log)) while not sync.ready(): try: cmd = self.inbox.get_nowait() if not self.process_command(cmd): # ctrl-c, basically! self.log.info("Stopping sync for {0}".format( self.email_address)) # make sure the parent can't start/stop any folder monitors # first sync.kill(block=True) killall(self.folder_monitors) return except Empty: sleep(self.heartbeat) assert not sync.successful(), \ "mail sync for {} account {} should run forever!"\ .format(self.provider, self.account_id) raise sync.exception def process_command(self, cmd): """ Returns True if successful, or False if process should abort. """ self.log.info("processing command {0}".format(cmd)) return cmd != 'shutdown' def _thread_finished(self, thread): state = getattr(thread, 'state') return state == 'finish' def _thread_polling(self, thread): state = getattr(thread, 'state') return state is not None and state.startswith('poll') def sync(self): raise NotImplementedError
class QueueCommandManager(CommandManager): """ A command manager that takes a queue of functions that act on the tracer and apply them one at a time with each call to next_command(). """ def __init__(self, tracer): super(QueueCommandManager, self).__init__(tracer) self.queue = Queue() self.sent = [] def enqueue(self, fn): """ Enqueues a new message to be consumed. """ self.queue.put(fn) def user_wait(self, duration): """ Simulates waiting on the user to feed us input for duration seconds. """ self.enqueue(lambda t: gevent.sleep(duration)) def clear(self): """ Clears the internal list of functions. """ self.queue = Queue() def user_next_command(self): """ Removes one message from the internal queue and apply it to the debugger. """ try: self.queue.get_nowait()(self.tracer) except Empty: return def send(self, msg): # Collect the output so that we can make assertions about it. self.sent.append(pickle.loads(msg)) user_stop = clear def start(self, auth_msg=''): pass
class MemorySession(Session): """ In memory session with a outgoing gevent Queue as the message store. """ def __init__(self, server, session_id=None): super(MemorySession, self).__init__(server, session_id=session_id) self.session_id = session_id or str(uuid.uuid4())[:8] self.server = server self.queue = Queue() self.hits = 0 self.heartbeats = 0 self.connected = False def add_message(self, msg): self.queue.put_nowait(msg) def get_messages(self, **kwargs): timeout = kwargs.get('timeout', None) self.incr_hits() if self.queue.empty(): try: return self.queue.get(**kwargs) except Empty: return [] else: accum = [] try: while not self.queue.empty(): if timeout: accum.append(self.queue.get(timeout=timeout)) else: accum.append(self.queue.get_nowait()) finally: return accum def interrupt(self): """ A kill event trigged through a client accessible endpoint Internal expires will not have is_interupted() == True """ self.interrupted = True self.kill() def kill(self): self.connected = False # Expire only once if not self.expired: self.expired = True self.timeout.set()
class NonBlockingIMap(object): ''' NonBlockingIMap class ''' def __init__(self, size=1, func=None, iterable=None): ''' __init__ ''' if size is not None and size <= 0: raise ValueError('size must not be negative and not equal zero: %r' % (size, )) self.size = size if func is None: raise RuntimeError('func must be assigned: %s' % (func, )) self.func = func self.greenlets_count = 0 self.iterable = iterable self.queue = Queue() def __iter__(self): ''' __iter__ ''' return self if PY3: __next__ = next del next def next(self): ''' next ''' gevent.sleep() value = None if self.greenlets_count >= self.size: return value try: item = self.iterable.get_nowait() gevent.spawn(self.func, item).link(self._on_result) self.greenlets_count += 1 except Empty: pass try: value = self.queue.get_nowait() except Empty: pass if not value and self.greenlets_count <= 0: raise StopIteration() return value def _on_result(self, greenlet): ''' _on_result ''' self.greenlets_count -= 1 if greenlet.successful(): self.queue.put(greenlet.value)
class WorkQueue(object): def __init__(self, worker, start_runner=None, max_work_load=16): if not worker: # TODO raise exception pass self.worker = worker self._start_runner = start_runner self._max_work_load = max_work_load # do we want to limit the size of the queue? self._queue = Queue() self._num_enqueues = 0 self._num_dequeues = 0 self._runner = Runner(self, self._max_work_load) #end __init__ def enqueue(self, work_item): self._queue.put(work_item) self._num_enqueues = self._num_enqueues + 1 self.may_be_start_runner() #end enqueue def dequeue(self): try: work_item = self._queue.get_nowait() except Empty: work_item = None else: self._num_dequeues = self._num_dequeues + 1 return work_item #end dequeue def may_be_start_runner(self): if self._queue.empty() or \ (self._start_runner and not self._start_runner()): return self._runner.start() #end may_be_start_runner def runner_done(self): if self._queue.empty() or \ (self._start_runner and not self._start_runner()): return True return False #end runner_done def is_queue_empty(self): if self._queue.empty(): return True return False def num_enqueues(self): return self._num_enqueues #end num_enqueues def num_dequeues(self): return self._num_dequeues
class AsyncPool(object): __slots__ = ["sockets", "socket_factory", "pool_size", "log", "local", "pid" ] def __init__(self, socket_factory, pool_size): self.pid = os.getpid() self.pool_size = pool_size self.socket_factory = socket_factory self.sockets = Queue() self.log = logging.getLogger('ming.async.AsyncPool') self.local = local() def _get_sock(self): return getattr(self.local, 'sock', None) def _set_sock(self, value): self.local.sock = value sock = property(_get_sock, _set_sock) def socket(self): pid = os.getpid() if pid != self.pid: self.sock = None self.sockets = Queue() self.pid = pid if self.sock is not None: self.log.debug('Return existing socket to greenlet %s', gevent.getcurrent() ) return self.sock gl = gevent.getcurrent() try: self.sock = self.sockets.get_nowait() self.log.debug('Checkout socket %s to greenlet %s', self.sock, gl ) except Empty: self.sock = self.socket_factory() self.log.debug('Create socket in greenlet %s', gl) self.sock.last_greenlet = gl return self.sock def return_socket(self): if self.sock is None: self.log.debug('No socket to return from greenlet %s', gevent.getcurrent() ) return if self.sockets.qsize() < self.pool_size: gl = gevent.getcurrent() self.log.debug('Checkin socket %s from greenlet %s', self.sock, gl) self.sockets.put(self.sock) self.sock = None else: self.log.debug('Close socket in greenlet %s', gevent.getcurrent() ) self.sock.close() self.sock = None self.local.sock = None
class Pipeline(object): """Persistent, bounded-memory-usage pipeline: - 'spills' items to disk if internal memory queue is filled-up. - restores queue contents from a journal file upon restart after crash. (planned, persistency is not implemented yet.) """ def __init__(self, receiver, logdir, size): """logdir: must exist """ self.receiver = receiver self.makedata = getattr(self.receiver, 'makedata', lambda o: o) self.logdir = logdir self.size = size self.queue = Queue(None) self.items = None self.seq = 0 def put(self, item): # TODO: write to journal data = self.makedata(item) self.queue.put((self.seq, data)) def _get_batch(self): # wait until at least one is available items = [self.queue.get()] # read more items if available try: while 1: item = self.queue.get_nowait() items.append(item) except Empty: pass return items def run(self): try: while 1: if self.items is None: self.items = self._get_batch() if self.items: try: data = [o[1] for o in self.items] self.receiver.put(data) # TODO: write to journal self.items = None except Exception, ex: logging.warn('%s.put() failed (%s), retrying...', self.receiver, ex, exc_info=1) gevent.sleep(5 * 60) except Exception as ex: logging.error('Pipeline.run exiting by error', exc_info=1)
class MemorySession(Session): """ In memory session with a outgoing gevent Queue as the message store. """ timer = 10.0 def __init__(self, server, session_id=None): super(MemorySession, self).__init__(server, session_id) self.session_id = session_id or str(uuid.uuid4())[:8] self.server = server self.queue = Queue() self.hits = 0 self.heartbeats = 0 self.connected = False def add_message(self, msg): self.queue.put_nowait(msg) def get_messages(self, **kwargs): self.incr_hits() if self.queue.empty(): try: return self.queue.get(**kwargs) except Empty: return [] else: accum = [] try: while not self.queue.empty(): accum.append(self.queue.get_nowait()) finally: return accum def interrupt(self): """ A kill event trigged through a client accessible endpoint Internal expires will not have is_interupted() == True """ self.interrupted = True self.kill() def kill(self): self.connected = False # Expire only once if not self.expired: self.expired = True self.timeout.set()
class geventimg(object): def __init__(self): self.file_path = 'D:/animeimg/animeimg2/' self.queue = Queue() def getanimeimgurlfrommysql(self): try: if not os.path.exists(self.file_path): print ('文件夹',self.file_path,'不存在,重新建立') os.makedirs(self.file_path) db.execute('SELECT a.ANIME_ID,a.ANIME_IMAGE FROM anime_home a',None) records = db.fetchall() if records: for r in records: self.queue.put_nowait(r) except Exception as e: pass else: pass finally: pass pass def downloadimg(self,n): while not self.queue.empty(): try: animeimg = self.queue.get_nowait() r = self.url_open('http://donghua.dmzj.com{}'.format(animeimg[1])) # print(animeimg) anime_img_name = self.file_path +'animepic_{}.jpg'.format(animeimg[0]) with open(anime_img_name,"wb") as f: f.write(r.content) f.flush() f.close() gevent.sleep(1) except Exception as e: traceback.print_exc() finally: pass def url_open(self,url): """ 爬取网页 """ req = request.get(url,5,'donghua.dmzj.com',None) return req def main(self): gevent.spawn(self.getanimeimgurlfrommysql).join() # gevent.spawn(self.downloadimg).join() pool = gevent.pool.Pool(5) threads = [] for r in range(5): threads.append(pool.spawn(self.downloadimg,r)) gevent.joinall(threads)
class ConnectionPool(object): def __init__(self, host, port, maxsize=10, connect_timeout=None, read_timeout=None, factory=lambda x: x): if not isinstance(maxsize, (int, long)): raise TypeError('Expected integer, got %r' % (maxsize, )) self.maxsize = maxsize self.pool = Queue() self.size = 0 self.host = host self.port = port self.factory = factory self.connect_timeout = connect_timeout self.read_timeout = read_timeout def get(self): pool = self.pool if self.size >= self.maxsize or pool.qsize(): return pool.get() else: self.size += 1 try: new_item = self.create_connection() except: self.size -= 1 raise return new_item def put(self, item): self.pool.put(item) def lose(self, item): self.size -= 1 item.close() def closeall(self): while not self.pool.empty(): conn = self.pool.get_nowait() try: conn.close() except Exception: pass def create_connection(self): """Create connection to remote host.""" sock = socket.create_connection((self.host, self.port), timeout=self.connect_timeout) sock.settimeout(self.read_timeout) return self.factory(sock)
def test_anonymous_no_concurrent_limit_because_remain_count( client, client_ip, mocker, url): mocker.patch.object(settings, 'CONCURRENT_LIMITER_SETTINGS', { '__anonymous__': { 'ttl': 100, 'capacity': 100, } }) def worker(queue): response = client.get(url) if response.status_code == 429: queue.put(429) greenlets = [] queue = Queue() for _ in range(3): greenlets.append(gevent.spawn(worker, queue)) gevent.joinall(greenlets) with raises(Empty): queue.get_nowait()
class Pipeline(object): """Persistent, bounded-memory-usage pipeline: - 'spills' items to disk if internal memory queue is filled-up. - restores queue contents from a journal file upon restart after crash. (planned, persistency is not implemented yet.) """ def __init__(self, receiver, logdir, size): """logdir: must exist """ self.receiver = receiver self.makedata = getattr(self.receiver, 'makedata', lambda o: o) self.logdir = logdir self.size = size self.queue = Queue(None) self.items = None self.seq = 0 def put(self, item): # TODO: write to journal data = self.makedata(item) self.queue.put((self.seq, data)) def _get_batch(self): # wait until at least one is available items = [self.queue.get()] # read more items if available try: while 1: item = self.queue.get_nowait() items.append(item) except Empty: pass return items def run(self): try: while 1: if self.items is None: self.items = self._get_batch() if self.items: try: data = [o[1] for o in self.items] self.receiver.put(data) # TODO: write to journal self.items = None except Exception, ex: logging.warn('%s.put() failed (%s), retrying...', self.receiver, ex, exc_info=1) gevent.sleep(5*60) except Exception as ex: logging.error('Pipeline.run exiting by error', exc_info=1)
def test_socket_enqueues_incoming_messages(): queue = Queue() request = testing.DummyRequest() request.registry['streamer.work_queue'] = queue socket = mock.Mock() client = websocket.WebSocket(socket) client.request = request message = FakeMessage('client data') client.received_message(message) result = queue.get_nowait() assert result.socket == client assert result.payload == 'client data'
class ASyncHandler(Group): def __init__(self, master, count=100, waitout=0.1): super(ASyncHandler, self).__init__( master, log=master.log, count=count, timeout=1) self.master = master self.queue = Queue() self.waitout = waitout self._size = 0 self._last = 0 def size(self): if time.time() - self._last < 1: return self._size self._last = time.time() self._size = self.queue.qsize() return self._size def do(self, func, *args, **kwargs): if self.size() < 500: self.queue.put((func, args, kwargs)) else: func(*args, **kwargs) def quick(self, limit=50): count = 0 while count < limit: try: func, args, kwargs = self.queue.get_nowait() func(*args, **kwargs) count += 1 except Empty: break def handle(self): count = 0 while True: try: func, args, kwargs = self.queue.get() func(*args, **kwargs) count += 1 except Empty: break self.wait(self.waitout) def on_quit(self): if not self.queue.empty(): self.log.info('%s - has %d tasks not done. now doing ...' % (self.name, self.queue.qsize()))
def test_process_nsq_topic_connects_reader_on_message_to_handle_message(get_reader): settings = {} queue = Queue() message = mock.Mock(body='hello') reader = get_reader.return_value reader.topic = 'donkeys' nsq.process_nsq_topic(settings, 'donkeys', queue, raise_error=False) message_handler = reader.on_message.connect.call_args[1]['receiver'] message_handler(reader, message) result = queue.get_nowait() assert result.topic == 'donkeys' assert result.payload == 'hello'
class ConnectionPool(object): def __init__(self, host, port, maxsize=10, connect_timeout=None, read_timeout=None, factory=lambda x: x): if not isinstance(maxsize, (int, long)): raise TypeError('Expected integer, got %r' % (maxsize, )) self.maxsize = maxsize self.pool = Queue() self.size = 0 self.host = host self.port = port self.factory = factory self.connect_timeout = connect_timeout self.read_timeout = read_timeout def get(self): pool = self.pool if self.size >= self.maxsize or pool.qsize(): return pool.get() else: self.size += 1 try: new_item = self.create_connection() except Exception: self.size -= 1 raise return new_item def put(self, item): self.pool.put(item) def lose(self, item): self.size -= 1 item.close() def closeall(self): while not self.pool.empty(): conn = self.pool.get_nowait() try: conn.close() except Exception: pass def create_connection(self): """Create connection to remote host.""" sock = socket.create_connection((self.host, self.port), timeout=self.connect_timeout) sock.settimeout(self.read_timeout) return self.factory(sock)
def test_handle_recv_event_puts_data_in_queue(self, gevent_mock, uwsgi_mock): recv_event = Event() recv_queue = Queue() recv_event.set() data = ['data1', 'data2', None] uwsgi_mock.websocket_recv_nb.side_effect = data self.handler.handle_recv_event(mock.Mock(), recv_event, recv_queue) self.assertTrue(gevent_mock.spawn.called) # Assert if recv_event is cleared self.assertFalse(recv_event.is_set()) self.assertEqual(len(recv_queue), 2) for d in data[:2]: self.assertEqual(recv_queue.get_nowait(), d)
def test_process_nsq_topic_connects_reader_on_message_to_handle_message( get_reader): settings = {} queue = Queue() message = mock.Mock(body='hello') reader = get_reader.return_value reader.topic = 'donkeys' nsq.process_nsq_topic(settings, 'donkeys', queue, raise_error=False) message_handler = reader.on_message.connect.call_args[1]['receiver'] message_handler(reader, message) result = queue.get_nowait() assert result.topic == 'donkeys' assert result.payload == 'hello'
class ConnectionPool(object): def __init__(self, connection_cls, maxsize=100, **kwargs): if not isinstance(maxsize, integer_types): raise TypeError('Expected integer, got %r' % (maxsize, )) self._connection_cls = connection_cls self._maxsize = maxsize self._pool = Queue() self._size = 0 self._conn_params = kwargs def get(self): if self._size >= self._maxsize or self._pool.qsize(): return self._pool.get() else: self._size += 1 try: return self._connection_cls(**self._conn_params) except: self._size -= 1 raise def put(self, item): self._pool.put(item) def closeall(self): while not self._pool.empty(): conn = self._pool.get_nowait() try: conn.close() except Exception: pass @contextlib.contextmanager def connection(self): conn = self.get() try: yield conn except: if conn.closed: conn = None self.closeall() raise finally: if conn is not None and not conn.closed: self.put(conn)
def test_anonymous_with_rate_limit(client, client_ip, mocker, configs, url): cfg = deepcopy(configs[0]) if '127.0.0.1' in cfg: cfg[client_ip] = cfg.pop('127.0.0.1') mocker.patch.object(settings, 'RATE_LIMITER_SETTINGS', cfg) def worker(queue): response = client.get(url) if response.status_code == 429: queue.put(429) greenlets = [] queue = Queue() for _ in range(5): greenlets.append(gevent.spawn(worker, queue)) gevent.joinall(greenlets) assert queue.get_nowait() == 429
class Node(object): node_id = None ip = None port = None pool = None def __init__(self, node_id, ip, port): self.node_id = node_id self.ip = ip self.port = port self.pool = Queue() @classmethod def create(cls, ip, port): return Node(uuid1(), ip, port) def __hash__(self): return long(self.node_id.hex, 16) def __eq__(self, other): return self.node_id == other.node_id def send(self, message): with self.connection as c: c.send(message) @property @contextmanager def connection(self): # returns a connection # to be used as # with node.connection as c: # do_stuff_with_c(c) try: connection = self.pool.get_nowait() except Empty: connection = Connection.connect(self.ip, self.port) yield connection self.pool.put(connection)
def event_stream(topic): '''Publishes events based on the topic.''' q = Queue() subscriptions[topic].append(q) try: while True: try: result = q.get_nowait() except gevent.queue.Empty: gevent.sleep(1) continue ev = ServerSentEvent(str(result)) yield ev.encode() except GeneratorExit: subscriptions[topic].remove(q) except Exception as ex: print "Generic event-stream exception!" print ex subscriptions[topic].remove(q)
class DatabaseConnectionPool(object): def __init__(self, maxsize=100): if not isinstance(maxsize, (int, long)): raise TypeError('Expected integer, got %r' % (maxsize, )) self.maxsize = maxsize self.pool = Queue() self.size = 0 def get(self): pool = self.pool if self.size >= self.maxsize or pool.qsize(): new_item = pool.get() try: # check connection is still valid self.check_usable(new_item) logger.debug("DB connection reused") except DatabaseError: logger.debug("DB connection was closed, creating new one") new_item = self.create_connection() return new_item else: self.size += 1 try: new_item = self.create_connection() logger.debug("DB connection created") except: self.size -= 1 raise return new_item def put(self, item): self.pool.put(item, timeout=2) def closeall(self): while not self.pool.empty(): conn = self.pool.get_nowait() try: conn.close() except Exception: pass self.size = 0
class Actor(gevent.Greenlet): def __init__(self): self.inbox = Queue() Greenlet.__init__(self) def receive(self, message): """ Define in your subclass. """ raise NotImplemented() def _run(self): self.running = True while self.running: try: message = self.inbox.get_nowait() self.receive(message) except Empty: pass gevent.sleep(0)
class MemorySession(session.Session): """ In memory session with a ``gevent.pool.Queue`` as the message store. """ def __init__(self, *args, **kwargs): super(MemorySession, self).__init__(*args, **kwargs) self._queue = Queue() def add_message(self, frame, data): log.info('session closed: %s', self.id) self._queue.put_nowait((frame, data)) #waiter = self._waiter #if waiter is not None: # self._waiter = None # if not waiter.cancelled(): # waiter.set_result(True) self._tick() def get_messages(self, timeout=None): self._tick() messages = [] # get all messages immediately pending in the queue while not self._queue.empty(): try: msg = self._queue.get_nowait() except Empty: break messages.append(msg) if not messages: try: messages.append(self._queue.get(timeout=timeout)) except Empty: pass return messages
def listen(): runtime = 0.0 rq = Queue(maxsize=None) stop_event = gevent.event.Event() gevent.spawn( self.ipool.listen_on, result_queue=rq, channel_name="notify_test_values", cancel_event=stop_event ) import time while 1: st = time.time() if runtime > 5.0: break try: notify = rq.get_nowait() print "NOTIFY", notify except QueueEmptyException: gevent.sleep(0.001) tt = time.time() - st runtime += tt stop_event.set() gevent.sleep(1)
def listen(): runtime = 0.0 rq = Queue(maxsize=None) stop_event = gevent.event.Event() gevent.spawn(self.ipool.listen_on, result_queue=rq, channel_name='notify_test_values', cancel_event=stop_event) import time while 1: st = time.time() if runtime > 5.0: break try: notify = rq.get_nowait() print "NOTIFY", notify except QueueEmptyException: gevent.sleep(0.001) tt = time.time() - st runtime += tt stop_event.set() gevent.sleep(1)
class BaseMailSyncMonitor(Greenlet): """ The SYNC_MONITOR_CLS for all mail sync providers should subclass this. Parameters ---------- account_id : int Which account to sync. email_address : str Email address for `account_id`. provider : str Provider for `account_id`. heartbeat : int How often to check for commands. retry_fail_classes : list Additional exceptions to *not* retry on. (This base class sets some defaults.) """ RETRY_FAIL_CLASSES = [MailsyncError, ValueError, AttributeError, DataError, IntegrityError, TypeError] def __init__(self, account, heartbeat=1, retry_fail_classes=[]): self.inbox = Queue() # how often to check inbox, in seconds self.heartbeat = heartbeat self.log = log.new(component='mail sync', account_id=account.id) self.account_id = account.id self.email_address = account.email_address self.provider_name = account.provider self.retry_fail_classes = self.RETRY_FAIL_CLASSES self.retry_fail_classes.extend(retry_fail_classes) # Stuff that might be updated later and we want to keep a shared # reference on child greenlets. if not hasattr(self, 'shared_state'): self.shared_state = dict() Greenlet.__init__(self) def _run(self): return retry_and_report_killed(self._run_impl, account_id=self.account_id, logger=self.log, fail_classes=self.retry_fail_classes) def _run_impl(self): sync = Greenlet(retry_and_report_killed, self.sync, account_id=self.account_id, logger=self.log, fail_classes=self.retry_fail_classes) sync.start() while not sync.ready(): try: cmd = self.inbox.get_nowait() if not self.process_command(cmd): # ctrl-c, basically! self.log.info("Stopping sync", email=self.email_address) # make sure the parent can't start/stop any folder monitors # first sync.kill(block=True) self.folder_monitors.kill() return except Empty: sleep(self.heartbeat) if sync.successful(): self.folder_monitors.kill() return # We just want the name of the exception so don't bother with # sys.exc_info() self.log.error('mail sync should run forever', provider=self.provider_name, account_id=self.account_id, exception=type(sync.exception).__name__) raise sync.exception def process_command(self, cmd): """ Returns True if successful, or False if process should abort. """ self.log.info("processing command {0}".format(cmd)) return cmd != 'shutdown' def _thread_finished(self, thread): state = getattr(thread, 'state') return state == 'finish' def _thread_polling(self, thread): state = getattr(thread, 'state') return state is not None and state.startswith('poll') def sync(self): raise NotImplementedError
__author__ = 'susperius' import logging import gevent import gevent.monkey from gevent.queue import Queue import worker.fuzzingworker as dbgworker from node.fuzzing.browser.javascript import JsDomFuzz gevent.monkey.patch_all() if __name__ == "__main__": logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.DEBUG) logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) fuz = JsDomFuzz(10, 1000, "ie") rep_q = Queue() #dbg_w = dbgworker.DebuggerWorker("C:\\Program Files\\Tracker Software\\PDF Viewer\\PDFXcview.exe\" -t \"testcases\\test.pdf", fuz, rep_q, 5) dbg_w = dbgworker.FuzzingWorker("C:\\Program Files\\Internet Explorer\\iexplore.exe", fuz, rep_q, 5, True) dbg_w.start_worker() gevent.sleep(20) rep = rep_q.get_nowait() print(rep) dbg_w.__create_testcases()
class Actor(Greenlet): """This is the base class for every node in the process tree. `Actor` manages the children of the various process nodes.""" def __init__(self, name): super(Actor, self).__init__() self.inbox = Queue() self.name = name self.running = False self._context = None # stores results of node if required # self._state = Message() # child nodes self._children = {} # listeners self.listeners = {} # all 'normal' messages go through self.process self.on(events.DATA, self.process) # on node start, execute preProcess self.on(events.START, self.preProcess) self.on(events.START, lambda msg: setattr(self, 'running', True)) # on node stop, set running to false, flush the queue and execute postProcess self.on(events.STOP, self.flush) self.on(events.STOP, self.postProcess) # self.on(events.STOP, lambda msg: setattr(self, 'running', False)) # self.on(events.STOP, self.kill) # on kill, call postProcess self.on(events.KILL, self.postProcess) self.on(events.KILL, lambda msg: setattr(self, 'running', False)) # self.on(events.KILL, self.kill) # listen to exceptions self.link_exception(self.handleException) def setContext(self, ctx): """Sets the execution context""" ctx.addActor(self) self._context = ctx for child in self.children: child.setContext(ctx) def getContext(self): """Gets the execution context""" return self._context def tick(self): """Yields the event loop to another node""" # self.log_trace("Yielding...") gevent.sleep(0) def sleep(self, seconds): """Makes the node sleep for the given seconds""" self.log_trace("Sleeping %ss..." % seconds) gevent.sleep(seconds) def handleException(self, exc): pass def stop(self): """Stop self and children""" self.handle(events.StopMessage) # gevent.joinall(list(self.children)) # self.tick() def start(self): """Starts the nodes""" if not self.running: self.handle(events.StartMessage) for child in self.children: child.start() super(Actor, self).start() # @property # def state(self): # """Returns the results from this processing element.""" # return self._state def clear(self): """Removes all children from this node""" self._children = {} def __len__(self): return len(self._children) def add(self, node, lazy=False): """Adds a child to the current node.""" validateType("node", (Actor, SynchronousActor), node) if not lazy: if node.name in self._children: raise Exception( "Same name siblings are not allowed. Child node, %s, already exists." % node.name) self._children[node.name] = (len(self._children), node) else: name = str(node.name) num = 0 while name in self: num += 1 name = (node.name + "-" + str(num)) node.name = name self._children[name] = (len(self._children), node) return self def __getitem__(self, name): """Returns the child with the name given. Raises NodeDoesNotExist exception if child does not exist.""" if not name in self._children: raise exceptions.NodeDoesNotExist(name) return self._children[name][1] def __contains__(self, name): """ Returns True if the node has the particular node. """ return name in self._children @property def children(self): """Returns a list of chilren in the order they were added.""" return (element for index, element in sorted(self._children.values())) def hasChildren(self): """Returns True if the node has any children.""" return len(self._children) > 0 def __delitem__(self, name): """Removes the child with the given name. Raises NodeDoesNotExist exception if the child does not exist.""" if not name in self._children: raise exceptions.NodeDoesNotExist(name) del self._children[name] def __isub__(self, child): del self[child] return self def __iadd__(self, child): return self.add(child) def __str__(self): return "<%s: name=\"%s\">" % (self.__class__.__name__, self.name) def __iter__(self): return self.children def preProcess(self, message=None): """Pre processes the data source. Can be overriden.""" pass def process(self, message=None): """This is the function which generates the results.""" raise NotImplementedError("process") def postProcess(self, message=None): """Post processes the data source. Can be overriden. Assumes inbox has been flushed and should close any open files / connections. """ pass def flush(self, message=None): """Empties the queue""" # gevent.joinall(self.dependencies) # waiting = True # while waiting: # waiting = False # for dep in self.dependencies: # if dep.running: # waiting = True # self.tick() self.log_debug("Flushing Queue...") if not self.inbox.empty(): for message in self.inbox: self.handle(message) self.running = False def execute(self): """Executes the preProcess, process, postProcess, scatter and gather methods""" self.running = True self.log_info("Starting...") while self.running: # self.log("Running...") try: message = self.inbox.get_nowait() self.handle(message) # yield to event loop after processing the message self.tick() except Empty: # Empty signifies that the queue is empty, so yield to another node self.tick() # pass # self.tick() # self.stop() self.log_info("Exiting...") def _run(self): self.execute() def scatter(self, message, forward=False): """Sends the results of this node to its children.""" # if self.hasChildren(): self.emit('data', message, forward) def emit(self, channel, message, forward=False): """The `emit` function can be used to send 'out-of-band' messages to any child nodes. This essentially allows a node to send special messages to any nodes listening.""" validateType('message', Message, message) message.channel = channel message.forward = forward # message.source = self self.log_debug("Sending message: %s on channel: %s" % (message, channel)) for child in self.children: child.insert(message) # child.inbox.put_nowait(message) # yields to event loop self.tick() def emitBatch(self, channel, messages, forward=False): """Add all messages to child queues before yielding""" for message in messages: message.channel = channel message.forward = forward # message.source = self self.log_debug("Sending message: %s on channel: %s" % (message, channel)) for child in self.children: # child.inbox.queue.extend(messages) child.insert(message) # for child in self.children: # child.inbox.queue.extend(messages) # yields to event loop self.tick() def removeListener(self, channel, function): """Removes a listener function from a channel""" if channel in self.listeners: try: self.listeners[channel].remove(function) return True except ValueError: return False else: return False def on(self, channel, function): """Registers event listeners based on a channel. Note: All functions registered should have 2 arguments. The first is current node (ie. self) and the second is the data being transferred. """ self.log_trace("Registering function on channel '%s'" % (channel)) if channel in self.listeners: self.listeners[channel].append(function) else: self.listeners[channel] = [function] def handle(self, message): """Handles events received on a given channel and forwards it if allowed.""" # logging message # if self.verbose: # self.log("Processing message: %s" % message) if message.channel in self.listeners: for function in self.listeners[message.channel]: # print function function(message) # forwards message if allowed if message.forward: self.log_trace("Forwarding message on channel: %s" % message.channel) for child in self.children: child.insert(message) # child.handle(message) # child.inbox.put_nowait(message) # self.tick() def insert(self, message): """Inserts a new message to be handled""" self.inbox.put_nowait(message) def log(self, msg): """Logging capability is baked into every Node.""" self.log_info(msg) def log_debug(self, msg): """Logs a DEBUG message""" log_debug(str(self.name).upper(), msg) def log_info(self, msg): """Logs an INFO message""" log_info(str(self.name).upper(), msg) def log_warning(self, msg): """Logs a WARNING message""" log_warning(str(self.name).upper(), msg) def log_error(self, msg): """Logs an ERROR message""" log_error(str(self.name).upper(), msg) def log_critical(self, msg): """Logs a CRITICAL message""" log_critical(str(self.name).upper(), msg) def log_exception(self, msg): """Logs an exception""" log_exception(str(self.name).upper(), msg) def log_trace(self, msg): """Logs low-level debug message""" log_trace(str(self.name).upper(), msg)
class Notifier(object): def __init__(self, app=None): self._url = None self._channel = None self._interval = 1 self._started = False self._q = Queue() self._logger = None if app: self.init_app(app) def init_app(self, app): self._url = app.config['SLACK_INCOMING_WEBHOOK'] self._channel = app.config['SLACK_CHANNEL'] self._interval = 1 / float(app.config['SLACK_RATE_LIMIT']) self._logger = app.logger spawn(self._loop) app.extensions['slack_notifier'] = self self._started = True self._logger.info( 'slack notifier started. min interval: %.1fs; default channel: %s', self._interval, self._channel) def notify(self, msg, channel=None, now=False): """ send a message to slack. The messages are buffered under the hood to avoid hitting Slack API rate limit. :param msg: message to send :param channel: channel to send the message """ if self._started: channel = channel if channel else self._channel if now: self._send(msg, channel) else: self._q.put((channel, msg)) def stop(self): self._started = False self.notify(id(self), None) # use id as token to stop def _send(self, message, channel): """ send to slack incoming webhook :param str message: message to send :param str channel: channel to send the message in :returns: True if send succeeds """ data = { 'text': message, 'parse': 'full', } if channel: data['channel'] = channel try: r = requests.post(self._url, data=json.dumps(data)) except (requests.ConnectionError, requests.Timeout) as e: self._logger.warning('Fail to send slack request: %s', e) return False else: if r.status_code == 200: return True else: self._logger.warning( 'Non-200 code returned from slack: %d - %s', r.status_code, r.content) return False def _flush(self): """ flush current queuing messages """ togo = defaultdict(list) try: while True: channel, msg = self._q.get_nowait() togo[channel].append(msg) except Empty: pass flushed = [] for channel, messages in togo.iteritems(): msg = '\n'.join(messages) if not self._send(msg, channel): self._logger.error('fail to send message to slack %s - %s', channel, msg) else: flushed.append((channel, msg)) return flushed def _loop(self): """ send loop """ last_send = time.time() while True: top = self._q.peek(block=True) if top == id(self): # stop break interval = time.time() - last_send if interval >= self._interval: # flush queue flushed = self._flush() last_send = time.time() self._logger.debug('flush finished at %.3f: %s', last_send, flushed) else: sleep(interval) self.notify('Bye.') self._flush()
class DatabaseConnectionPool(object): def __init__(self, maxsize=100): if not isinstance(maxsize, (int, long)): raise TypeError('Expected integer, got %r' % (maxsize, )) self.maxsize = maxsize self.pool = Queue() self.size = 0 def get(self): pool = self.pool if self.size >= self.maxsize or pool.qsize(): return pool.get() else: self.size += 1 try: new_item = self.create_connection() except: self.size -= 1 raise return new_item def put(self, item): self.pool.put(item) def closeall(self): while not self.pool.empty(): conn = self.pool.get_nowait() try: conn.close() except Exception: pass @contextlib.contextmanager def connection(self, isolation_level=None): conn = self.get() try: if isolation_level is not None: if conn.isolation_level == isolation_level: isolation_level = None else: conn.set_isolation_level(isolation_level) yield conn except: if conn.closed: conn = None self.closeall() else: conn = self._rollback(conn) raise else: if conn.closed: raise OperationalError("Cannot commit because connection was closed: %r" % (conn, )) conn.commit() finally: if conn is not None and not conn.closed: if isolation_level is not None: conn.set_isolation_level(isolation_level) self.put(conn) @contextlib.contextmanager def cursor(self, *args, **kwargs): isolation_level = kwargs.pop('isolation_level', None) conn = self.get() try: if isolation_level is not None: if conn.isolation_level == isolation_level: isolation_level = None else: conn.set_isolation_level(isolation_level) yield conn.cursor(*args, **kwargs) except: if conn.closed: conn = None self.closeall() else: conn = self._rollback(conn) raise else: if conn.closed: raise OperationalError("Cannot commit because connection was closed: %r" % (conn, )) conn.commit() finally: if conn is not None and not conn.closed: if isolation_level is not None: conn.set_isolation_level(isolation_level) self.put(conn) def _rollback(self, conn): try: conn.rollback() except: gevent.get_hub().handle_error(conn, *sys.exc_info()) return return conn def execute(self, *args, **kwargs): with self.cursor(**kwargs) as cursor: cursor.execute(*args) return cursor.rowcount def fetchone(self, *args, **kwargs): with self.cursor(**kwargs) as cursor: cursor.execute(*args) return cursor.fetchone() def fetchall(self, *args, **kwargs): with self.cursor(**kwargs) as cursor: cursor.execute(*args) return cursor.fetchall() def fetchiter(self, *args, **kwargs): with self.cursor(**kwargs) as cursor: cursor.execute(*args) while True: items = cursor.fetchmany() if not items: break for item in items: yield item
class WorkQueue(object): _MAX_QUEUE_SIZE = 1024 _MAX_WORKLOAD = 16 def __init__(self, worker, start_runner=None, max_qsize=None, max_work_load=None): self.worker = worker self._start_runner = start_runner self._max_qsize = max_qsize or WorkQueue._MAX_QUEUE_SIZE self._max_work_load = max_work_load or WorkQueue._MAX_WORKLOAD self._bounded = False self._queue = Queue() self._qsize = 0 self._num_enqueues = 0 self._num_dequeues = 0 self._drops = 0 self._high_watermarks = None self._low_watermarks = None self._hwm_index = -1 self._lwm_index = -1 self._runner = Runner(self, self._max_work_load) # end __init__ def set_bounded(self, bounded): self._bounded = bounded # end set_bounded def bounded(self): return self._bounded # end bounded def set_high_watermarks(self, high_wm): # weed out duplicates and store the watermarks in sorted order self._high_watermarks = list(sorted(set(high_wm))) self._set_watermark_indices(-1, -1) # end set_high_watermarks def high_watermarks(self): return self._high_watermarks # end high_watermarks def set_low_watermarks(self, low_wm): # weed out duplicates and store the watermarks in sorted order self._low_watermarks = list(sorted(set(low_wm))) self._set_watermark_indices(-1, -1) # end set_low_watermarks def low_watermarks(self): return self._low_watermarks # end low_watermarks def watermark_indices(self): return self._hwm_index, self._lwm_index # end watermark_indices def enqueue(self, work_item): self.increment_queue_size(work_item) if self._bounded: if self._qsize > self._max_qsize: self.decrement_queue_size(work_item) self._drops += 1 return False self._num_enqueues += 1 self._process_high_watermarks() self._queue.put(work_item) self.may_be_start_runner() return True # end enqueue def dequeue(self): try: work_item = self._queue.get_nowait() except Empty: work_item = None else: self.decrement_queue_size(work_item) self._num_dequeues += 1 self._process_low_watermarks() return work_item # end dequeue def increment_queue_size(self, work_item): self._qsize += 1 # end increment_queue_size def decrement_queue_size(self, work_item): self._qsize -= 1 # end decrement_queue_size def size(self): return self._qsize # end size def may_be_start_runner(self): if self._queue.empty() or \ (self._start_runner and not self._start_runner()): return self._runner.start() # end may_be_start_runner def runner_done(self): if self._queue.empty() or \ (self._start_runner and not self._start_runner()): return True return False # end runner_done def is_queue_empty(self): if self._queue.empty(): return True return False # end is_queue_empty def num_enqueues(self): return self._num_enqueues # end num_enqueues def num_dequeues(self): return self._num_dequeues # end num_dequeues def drops(self): return self._drops # end drops def runner(self): return self._runner # end runner def _set_watermark_indices(self, hwm_index, lwm_index): self._hwm_index = hwm_index self._lwm_index = lwm_index # end _set_watermark_indices def _process_high_watermarks(self): if not self._high_watermarks: return # Check if we have crossed any high watermarks. # Find the index of the first element greater than self._qsize # in self._high_watermarks. index = bisect.bisect_right(self._high_watermarks, WaterMark(self._qsize, None)) # If the first element > qsize, then we have not crossed any # high watermark. if index == 0: return # We have crossed (index-1)th watermark in the list. hwm_index = index - 1 if hwm_index == self._hwm_index: return self._set_watermark_indices(hwm_index, hwm_index + 1) # Now invoke the watermark callback self._high_watermarks[self._hwm_index].callback(self._qsize) # end _process_high_watermarks def _process_low_watermarks(self): if not self._low_watermarks: return # Check if we have crossed any low watermarks. # Find the index of the first element not less than self._qsize # in self._low_watermarks. index = bisect.bisect_left(self._low_watermarks, WaterMark(self._qsize, None)) # If there is no element >= qsize, then we have not crossed any # low watermark. if index == len(self._low_watermarks): return lwm_index = index if lwm_index == self._lwm_index: return self._set_watermark_indices(lwm_index - 1, lwm_index) # Now invoke the watermark callback self._low_watermarks[self._lwm_index].callback(self._qsize)
class Connection(object): in_buffer_size = 4096 out_buffer_size = 4096 cql_version = None keyspace = None compression = True compressor = None decompressor = None ssl_options = None last_error = None in_flight = 0 is_defunct = False is_closed = False lock = None def __init__( self, host="127.0.0.1", port=9042, credentials=None, ssl_options=None, sockopts=None, compression=True, cql_version=None, ): self.host = host self.port = port self.credentials = credentials self.ssl_options = ssl_options self.sockopts = sockopts self.compression = compression self.cql_version = cql_version self._id_queue = Queue(MAX_STREAM_PER_CONNECTION) for i in range(MAX_STREAM_PER_CONNECTION): self._id_queue.put_nowait(i) self.lock = RLock() def close(self): raise NotImplementedError() def defunct(self, exc): with self.lock: if self.is_defunct or self.is_closed: return self.is_defunct = True trace = traceback.format_exc(exc) if trace != "None": log.debug("Defuncting connection (%s) to %s: %s\n%s", id(self), self.host, exc, traceback.format_exc(exc)) else: log.debug("Defuncting connection (%s) to %s: %s", id(self), self.host, exc) self.last_error = exc self.close() self.error_all_callbacks(exc) self.connected_event.set() return exc def error_all_callbacks(self, exc): with self.lock: callbacks = self._callbacks self._callbacks = {} new_exc = ConnectionShutdown(str(exc)) for cb in callbacks.values(): try: cb(new_exc) except Exception: log.warn( "Ignoring unhandled exception while erroring callbacks for a " "failed connection (%s) to host %s:", id(self), self.host, exc_info=True, ) def handle_pushed(self, response): log.debug("Message pushed from server: %r", response) for cb in self._push_watchers.get(response.event_type, []): try: cb(response.event_args) except Exception: log.exception("Pushed event handler errored, ignoring:") def send_msg(self, msg, cb, wait_for_id=False): if self.is_defunct: raise ConnectionShutdown("Connection to %s is defunct" % self.host) elif self.is_closed: raise ConnectionShutdown("Connection to %s is closed" % self.host) if not wait_for_id: try: request_id = self._id_queue.get_nowait() except Queue.Empty: raise ConnectionBusy("Connection to %s is at the max number of requests" % self.host) else: request_id = self._id_queue.get() self._callbacks[request_id] = cb self.push(msg.to_string(request_id, compression=self.compressor)) return request_id def wait_for_response(self, msg, timeout=None): return self.wait_for_responses(msg, timeout=timeout)[0] def wait_for_responses(self, *msgs, **kwargs): timeout = kwargs.get("timeout") waiter = ResponseWaiter(self, len(msgs)) # busy wait for sufficient space on the connection messages_sent = 0 while True: needed = len(msgs) - messages_sent with self.lock: available = min(needed, MAX_STREAM_PER_CONNECTION - self.in_flight) self.in_flight += available for i in range(messages_sent, messages_sent + available): self.send_msg(msgs[i], partial(waiter.got_response, index=i), wait_for_id=True) messages_sent += available if messages_sent == len(msgs): break else: if timeout is not None: timeout -= 0.01 if timeout <= 0.0: raise OperationTimedOut() time.sleep(0.01) try: return waiter.deliver(timeout) except OperationTimedOut: raise except Exception, exc: self.defunct(exc) raise
class Server(object): def __init__(self, address, size=None, log_level=DEFAULT_LOG_LEVEL): self.daemon = True self.started = False self.size = size self.queue = Queue(maxsize=size) self.address = address self.context = zmq.Context(1) self.server = None self.logger = get_logger(self, log_level) self._has_fetched_jobs = False def send(self, cmd, data=''): self.server.send_multipart([cmd, data]) def recv(self): reply = self.server.recv_multipart() assert len(reply) == 2 return reply def bind(self): if self.server: self.server.close() self.server = self.context.socket(zmq.REP) self.server.bind(self.address) def start(self): self.started = True self.logger.info("Taskmaster binding to %r", self.address) self.bind() while self.started: gevent.sleep(0) cmd, data = self.recv() if cmd == 'GET': if not self.has_work(): self.send('QUIT') continue try: job = self.queue.get_nowait() except Empty: self.send('WAIT') continue self.send('OK', pickle.dumps(job)) elif cmd == 'DONE': self.queue.task_done() if self.has_work(): self.send('OK') else: self.send('QUIT') else: self.send('ERROR', 'Unrecognized command') self.logger.info('Shutting down') self.shutdown() def mark_queue_filled(self): self._has_fetched_jobs = True def put_job(self, job): return self.queue.put(job) def first_job(self): return self.queue.queue[0] def get_current_size(self): return self.queue.qsize() def get_max_size(self): return self.size def has_work(self): if not self._has_fetched_jobs: return True return not self.queue.empty() def is_alive(self): return self.started def shutdown(self): if not self.started: return self.server.close() self.context.term() self.started = False
class WebSocket: def __init__(self, socket): self.socket = socket self.closed = False self.status = None self._receive_error = None self._queue = Queue() self.max_length = 10 * 1024 * 1024 gevent.spawn(self._listen) def set_max_message_length(self, length): self.max_length = length def _listen(self): try: while True: fin = False message = bytearray() is_first_message = True start_opcode = None while not fin: payload, opcode, fin = self._get_frame(max_length=self.max_length - len(message)) # Make sure continuation frames have correct information if not is_first_message and opcode != 0: self._error(STATUS_PROTOCOL_ERROR) if is_first_message: if opcode not in (OPCODE_TEXT, OPCODE_BINARY): self._error(STATUS_PROTOCOL_ERROR) # Save opcode start_opcode = opcode message += payload is_first_message = False message = bytes(message) if start_opcode == OPCODE_TEXT: # UTF-8 text try: message = message.decode() except UnicodeDecodeError: self._error(STATUS_DATA_ERROR) self._queue.put(message) except Exception as e: self.closed = True self._receive_error = e self._queue.put(None) # To make sure the error is read def receive(self): if not self._queue.empty(): return self.receive_nowait() if isinstance(self._receive_error, EOFError): return None if self._receive_error: raise self._receive_error self._queue.peek() return self.receive_nowait() def receive_nowait(self): ret = self._queue.get_nowait() if self._receive_error and not isinstance(self._receive_error, EOFError): raise self._receive_error return ret def send(self, data): if self.closed: raise EOFError() if isinstance(data, str): self._send_frame(OPCODE_TEXT, data.encode()) elif isinstance(data, bytes): self._send_frame(OPCODE_BINARY, data) else: raise TypeError("Expected str or bytes, got " + repr(type(data))) # Reads a frame from the socket. Pings, pongs and close packets are handled # automatically def _get_frame(self, max_length): while True: payload, opcode, fin = self._read_frame(max_length=max_length) if opcode == OPCODE_PING: self._send_frame(OPCODE_PONG, payload) elif opcode == OPCODE_PONG: pass elif opcode == OPCODE_CLOSE: if len(payload) >= 2: self.status = struct.unpack("!H", payload[:2])[0] was_closed = self.closed self.closed = True if not was_closed: # Send a close frame in response self.close(STATUS_OK) raise EOFError() else: return payload, opcode, fin # Low-level function, use _get_frame instead def _read_frame(self, max_length): header = self._recv_exactly(2) if not (header[1] & 0x80): self._error(STATUS_POLICY_VIOLATION) opcode = header[0] & 0xf fin = bool(header[0] & 0x80) payload_length = header[1] & 0x7f if payload_length == 126: payload_length = struct.unpack("!H", self._recv_exactly(2))[0] elif payload_length == 127: payload_length = struct.unpack("!Q", self._recv_exactly(8))[0] # Control frames are handled in a special way if opcode in (OPCODE_PING, OPCODE_PONG): max_length = 125 if payload_length > max_length: self._error(STATUS_TOO_LONG) mask = self._recv_exactly(4) payload = self._recv_exactly(payload_length) payload = self._unmask(payload, mask) return payload, opcode, fin def _recv_exactly(self, length): buf = bytearray() while len(buf) < length: block = self.socket.recv(min(4096, length - len(buf))) if block == b"": raise EOFError() buf += block return bytes(buf) def _unmask(self, payload, mask): def gen(c): return bytes([x ^ c for x in range(256)]) payload = bytearray(payload) payload[0::4] = payload[0::4].translate(gen(mask[0])) payload[1::4] = payload[1::4].translate(gen(mask[1])) payload[2::4] = payload[2::4].translate(gen(mask[2])) payload[3::4] = payload[3::4].translate(gen(mask[3])) return bytes(payload) def _send_frame(self, opcode, data): for i in range(0, len(data), SEND_PACKET_SIZE): part = data[i:i + SEND_PACKET_SIZE] fin = int(i == (len(data) - 1) // SEND_PACKET_SIZE * SEND_PACKET_SIZE) header = bytes( [ (opcode if i == 0 else 0) | (fin << 7), min(len(part), 126) ] ) if len(part) >= 126: header += struct.pack("!H", len(part)) self.socket.sendall(header + part) def _error(self, status): self.close(status) raise EOFError() def close(self, status=STATUS_OK): self.closed = True try: self._send_frame(OPCODE_CLOSE, struct.pack("!H", status)) except (BrokenPipeError, ConnectionResetError): pass self.socket.close()
class AbstractDatabaseConnectionPool(object): def __init__(self, maxsize=100): if not isinstance(maxsize, integer_types): raise TypeError('Expected integer, got %r' % (maxsize, )) self.maxsize = maxsize self.pool = Queue() self.size = 0 def create_connection(self): raise NotImplementedError() def get(self): pool = self.pool if self.size >= self.maxsize or pool.qsize(): return pool.get() self.size += 1 try: new_item = self.create_connection() except: self.size -= 1 raise return new_item def put(self, item): self.pool.put(item) def closeall(self): while not self.pool.empty(): conn = self.pool.get_nowait() try: conn.close() except Exception: pass @contextlib.contextmanager def connection(self, isolation_level=None): conn = self.get() try: if isolation_level is not None: if conn.isolation_level == isolation_level: isolation_level = None else: conn.set_isolation_level(isolation_level) yield conn except: if conn.closed: conn = None self.closeall() else: conn = self._rollback(conn) raise else: if conn.closed: raise OperationalError( "Cannot commit because connection was closed: %r" % (conn, )) conn.commit() finally: if conn is not None and not conn.closed: if isolation_level is not None: conn.set_isolation_level(isolation_level) self.put(conn) @contextlib.contextmanager def cursor(self, *args, **kwargs): isolation_level = kwargs.pop('isolation_level', None) with self.connection(isolation_level) as conn: yield conn.cursor(*args, **kwargs) def _rollback(self, conn): try: conn.rollback() except: gevent.get_hub().handle_error(conn, *sys.exc_info()) return return conn def execute(self, *args, **kwargs): with self.cursor(**kwargs) as cursor: cursor.execute(*args) return cursor.rowcount def fetchone(self, *args, **kwargs): with self.cursor(**kwargs) as cursor: cursor.execute(*args) return cursor.fetchone() def fetchall(self, *args, **kwargs): with self.cursor(**kwargs) as cursor: cursor.execute(*args) return cursor.fetchall() def fetchiter(self, *args, **kwargs): with self.cursor(**kwargs) as cursor: cursor.execute(*args) while True: items = cursor.fetchmany() if not items: break for item in items: yield item
class DatabaseConnectionPool(object): """ Gevent compliant database connection pool """ def __init__(self, maxsize=100): if not isinstance(maxsize, (int, long)): raise TypeError('Expected integer, got %r' % (maxsize, )) self.maxsize = maxsize # Maximum connections (pool + checkout out) self.pool = Queue() # Open connection pool self.size = 0 # Number of open connections def get(self): pool = self.pool if self.size >= self.maxsize or pool.qsize(): return pool.get() else: self.size += 1 try: new_item = self.create_connection() except: self.size -= 1 raise return new_item def put(self, item): self.pool.put(item) def closeall(self): while not self.pool.empty(): conn = self.pool.get_nowait() try: conn.close() self.size -= 1 except Exception: pass @contextlib.contextmanager def in_transaction(self, isolation_level=None): trans_conn = getattr(db_context, "cur_transaction", None) if trans_conn: raise OperationalError("Already in a transaction context") conn = self.get() db_context.cur_transaction = conn try: if isolation_level is not None: if conn.isolation_level == isolation_level: isolation_level = None else: conn.set_isolation_level(isolation_level) yield conn except: if conn.closed: conn = None self.closeall() else: conn = self._rollback(conn) raise else: if conn.closed: raise OperationalError( "Cannot commit because connection was closed: %r" % (conn, )) conn.commit() finally: if conn is not None and not conn.closed: if isolation_level is not None: conn.set_isolation_level(isolation_level) self.put(conn) db_context.cur_transaction = None @contextlib.contextmanager def connection(self, isolation_level=None): trans_conn = getattr(db_context, "cur_transaction", None) conn = trans_conn if trans_conn else self.get() try: if isolation_level is not None: if conn.isolation_level == isolation_level: isolation_level = None else: conn.set_isolation_level(isolation_level) yield conn except: if conn.closed: conn = None self.closeall() else: conn = self._rollback(conn) raise else: if conn.closed: raise OperationalError( "Cannot commit because connection was closed: %r" % (conn, )) if not trans_conn: conn.commit() finally: if conn is not None and not conn.closed: if isolation_level is not None: conn.set_isolation_level(isolation_level) if not trans_conn: self.put(conn) @contextlib.contextmanager def cursor(self, *args, **kwargs): isolation_level = kwargs.pop('isolation_level', None) trans_conn = getattr(db_context, "cur_transaction", None) conn = trans_conn if trans_conn else self.get() try: if isolation_level is not None: if conn.isolation_level == isolation_level: isolation_level = None else: conn.set_isolation_level(isolation_level) tracer = kwargs.pop("tracer", None) cur = conn.cursor(*args, **kwargs) if isinstance(cur, TracingCursor): cur._tracer = tracer yield cur except: if conn.closed: conn = None self.closeall() else: conn = self._rollback(conn) raise else: if conn.closed: raise OperationalError( "Cannot commit because connection was closed: %r" % (conn, )) if not trans_conn: conn.commit() finally: if conn is not None and not conn.closed: if isolation_level is not None: conn.set_isolation_level(isolation_level) if not trans_conn: self.put(conn) def _rollback(self, conn): try: conn.rollback() except: gevent.get_hub().handle_error(conn, *sys.exc_info()) return return conn def execute(self, *args, **kwargs): with self.cursor(**kwargs) as cursor: cursor.execute(*args) return cursor.rowcount def fetchone(self, *args, **kwargs): with self.cursor(**kwargs) as cursor: cursor.execute(*args) return cursor.fetchone() def fetchall(self, *args, **kwargs): with self.cursor(**kwargs) as cursor: cursor.execute(*args) return cursor.fetchall() def fetchiter(self, *args, **kwargs): with self.cursor(**kwargs) as cursor: cursor.execute(*args) while True: items = cursor.fetchmany() if not items: break for item in items: yield item
class Actor(object): """ Class that contains a queue and a greenlet serving that queue. """ max_ops_before_yield = 10000 """Number of calls to self._maybe_yield before it yields""" def __init__(self, qualifier=None): self._event_queue = Queue() self.greenlet = gevent.Greenlet(self._loop) self._op_count = 0 self._current_msg = None self.started = False # Message being processed; purely for logging. self.msg_uuid = None # Logging parameters self.qualifier = qualifier if qualifier: self.name = "%s(%s)" % (self.__class__.__name__, qualifier) else: self.name = self.__class__.__name__ # Can't use str(self) yet, it might not be ready until subclass # constructed. _log.info("%s created.", self.name) def start(self): assert not self.greenlet, "Already running" _log.info("Starting %s", self) self.started = True self.greenlet.start() return self def _loop(self): """ Main greenlet loop, repeatedly runs _step(). Doesn't return normally. """ actor_storage.name = self.name actor_storage.msg_uuid = None try: while True: self._step() except: _log.exception("Exception killed %s", self) raise def _step(self): """ Run one iteration of the event loop for this actor. Mainly broken out to allow the UTs to single-step an Actor. It also has the beneficial side effect of introducing a new local scope so that our variables die before we block next time. """ # Block waiting for work. msg = self._event_queue.get() actor_storage.msg_uuid = msg.uuid batch = [msg] batches = [] if not msg.needs_own_batch: # Try to pull some more work off the queue to combine into a # batch. while not self._event_queue.empty(): # We're the only ones getting from the queue so this should # never fail. msg = self._event_queue.get_nowait() if msg.needs_own_batch: if batch: batches.append(batch) batches.append([msg]) batch = [] else: batch.append(msg) if batch: batches.append(batch) num_splits = 0 while batches: # Process the first batch on our queue of batches. Invariant: # we'll either process this batch to completion and discard it or # we'll put all the messages back into the batch queue in the same # order but with a first batch that is half the size and the # rest of its messages in the second batch. batch = batches.pop(0) # Give subclass a chance to filter the batch/update its state. batch = self._start_msg_batch(batch) assert batch is not None, "_start_msg_batch() should return batch." results = [] # Will end up same length as batch. for msg in batch: _log.debug("Message %s recd by %s from %s, queue length %d", msg, msg.recipient, msg.caller, self._event_queue.qsize()) self._current_msg = msg try: # Actually execute the per-message method and record its # result. result = msg.method() except BaseException as e: _log.exception("Exception processing %s", msg) results.append(ResultOrExc(None, e)) else: results.append(ResultOrExc(result, None)) finally: self._current_msg = None try: # Give subclass a chance to post-process the batch. _log.debug("Finishing message batch") self._finish_msg_batch(batch, results) except SplitBatchAndRetry: # The subclass couldn't process the batch as is (probably # because a failure occurred and it couldn't figure out which # message caused the problem). Split the batch into two and # re-run it. _log.warn("Splitting batch to retry.") self.__split_batch(batch, batches) num_splits += 1 # For diags. continue except BaseException as e: # Most-likely a bug. Report failure to all callers. _log.exception("_finish_msg_batch failed.") results = [(None, e)] * len(results) # Batch complete and finalized, set all the results. assert len(batch) == len(results) for msg, (result, exc) in zip(batch, results): for future in msg.results: if exc is not None: future.set_exception(exc) else: future.set(result) if num_splits > 0: _log.warn("Split batches complete. Number of splits: %s", num_splits) @staticmethod def __split_batch(current_batch, remaining_batches): """ Splits batch in half and prepends it to the list of remaining batches. Modifies remaining_batches in-place. :param list[Message] current_batch: list of messages that's currently being processed. :param list[list[Message]] remaining_batches: list of batches still to process. """ assert len(current_batch) > 1, "Batch too small to split" # Split the batch. split_point = len(current_batch) // 2 _log.debug("Split-point = %s", split_point) first_half = current_batch[:split_point] second_half = current_batch[split_point:] if remaining_batches and not remaining_batches[0][0].needs_own_batch: # Optimization: there's another batch already queued and # it also contains batchable messages push the second # half of this batch onto the front of that one. _log.debug("Split batch and found a subsequent batch, " "coalescing with that.") next_batch = remaining_batches[0] next_batch[:0] = second_half else: _log.debug("Split batch but cannot prepend to next batch, adding " "both splits to start of queue.") remaining_batches[:0] = [second_half] remaining_batches[:0] = [first_half] def _start_msg_batch(self, batch): """ Called before processing a batch of messages to give subclasses a chance to filter the batch. Implementations must ensure that every AsyncResult in the batch is correctly set. Usually, that means combining them into one list. It is usually easier to build up a batch of changes to make in the @actor_message-decorated methods and then process them in _finish_msg_batch(). Intended to be overridden. This implementation simply returns the input batch. :param list[Message] batch: """ return batch def _finish_msg_batch(self, batch, results): """ Called after a batch of events have been processed from the queue before results are set. Intended to be overridden. This implementation does nothing. Exceptions raised by this method are propagated to all messages in the batch, overriding the existing results. It is recommended that the implementation catches appropriate exceptions and maps them back to the correct entry in results. :param list[ResultOrExc] results: Pairs of (result, exception) representing the result of each message-processing function. Only one of the values is set. Updates to the list alter the result send to any waiting listeners. :param list[Message] batch: The input batch, always the same length as results. """ pass def _maybe_yield(self): """ With some probability, yields processing to another greenlet. (Utility method to be called from the actor's greenlet during long-running operations.) """ self._op_count += 1 if self._op_count >= self.max_ops_before_yield: gevent.sleep() self._op_count = 0 def __str__(self): return self.__class__.__name__ + "<queue_len=%s,live=%s,msg=%s>" % ( self._event_queue.qsize(), bool(self.greenlet), self._current_msg )
class OutputBatch(ActorBaseFT): def __init__(self, name, chassis, config): self._queue = None super(OutputBatch, self).__init__(name, chassis, config) self._push_glet = None self._checkpoint_glet = None def configure(self): super(OutputBatch, self).configure() self.queue_maxsize = int(self.config.get('queue_maxsize', 100000)) if self.queue_maxsize == 0: self.queue_maxsize = None self._queue = Queue(maxsize=self.queue_maxsize) self.client_id = self.config.get('client_id', None) self.client_secret = self.config.get('client_secret', None) self.tenant_id = self.config.get('tenant_id', None) self.action = self.config.get('action', 'Alert') self.severity = self.config.get('severity', None) self.side_config_path = self.config.get('side_config', None) if self.side_config_path is None: self.side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '%s_side_config.yml' % self.name) self._load_side_config() def _load_side_config(self): try: with open(self.side_config_path, 'r') as f: sconfig = yaml.safe_load(f) except Exception as e: LOG.error('%s - Error loading side config: %s', self.name, str(e)) return client_id = sconfig.get('client_id', None) if client_id is not None: self.client_id = client_id LOG.info('{} - client_id set'.format(self.name)) client_secret = sconfig.get('client_secret', None) if client_secret is not None: self.client_secret = client_secret LOG.info('{} - client_secret set'.format(self.name)) tenant_id = sconfig.get('tenant_id', None) if tenant_id is not None: self.tenant_id = tenant_id LOG.info('{} - tenant_id set'.format(self.name)) action = sconfig.get('action', None) if action is not None: self.action = action LOG.info('{} - action set'.format(self.action)) def connect(self, inputs, output): output = False super(OutputBatch, self).connect(inputs, output) def _initialize_table(self, truncate=False): self.table = Table(name=self.name, truncate=truncate) def initialize(self): self._initialize_table() def rebuild(self): self._initialize_table(truncate=(self.last_checkpoint is None)) def reset(self): self._initialize_table(truncate=True) def _get_auth_token(self): if self.client_id is None: LOG.error('{} - client_id not set'.format(self.name)) raise AuthConfigException('{} - client_id not set'.format( self.name)) if self.client_secret is None: LOG.error('{} - client_secret not set'.format(self.name)) raise AuthConfigException('{} - client_secret not set'.format( self.name)) if self.tenant_id is None: LOG.error('{} - tenant_id not set'.format(self.name)) raise AuthConfigException('{} - tenant_id not set'.format( self.name)) context = adal.AuthenticationContext( AUTHORITY_URL.format(self.tenant_id), validate_authority=self.tenant_id != 'adfs', api_version=None) token = context.acquire_token_with_client_credentials( RESOURCE, self.client_id, self.client_secret) if token is None or 'accessToken' not in token: LOG.error('{} - Invalid token or accessToken not available'.format( self.name)) raise RuntimeError( '{} - Invalid token or accessToken not available'.format( self.name)) return token['accessToken'] def _push_indicators(self, token, indicators): message = {'Indicators': list(indicators)} LOG.debug(message) result = requests.post(WD_ATP_TIINDICATORS_ENDPOINT, headers={ 'Content-Type': 'application/json', 'Authorization': 'Bearer {}'.format(token) }, json=message) LOG.debug(result.text) result.raise_for_status() # Check the status of the submitted indicators # NOTE: if the indicator contains a range split by _encode_indicators, a partial submission might go through # i.e. 192.168.0.1-192.168.0.3 can be split in 192.168.0.1, 192.168.0.2 and 192.168.0.3 # the first might go through, the second might return error # This output node doesn't check for this condition (although the error counters are correctly updated) result = result.json() if not result or '@odata.context' not in result or result[ '@odata.context'] != 'https://api.securitycenter.windows.com/api/$metadata#Collection(microsoft.windowsDefenderATP.api.ImportIndicatorResult)': raise WDATPResponseException('Unexpected response from WDATP API') if 'value' not in result: raise WDATPResponseException('Missing value from WDATP API result') for v in result['value']: if 'indicator' not in v or 'isFailed' not in v: raise WDATPResponseException( 'Missing indicator values from WDATP response') LOG.debug( '{} - Got result for indicator {}: isFailed is {}'.format( self.name, v['indicator'], v["isFailed"])) if not v["isFailed"]: # Success! self.statistics['indicator.tx'] += 1 else: failReason = v[ 'failureReason'] if 'failureReason' in v else 'Unknown' LOG.error('{}: error submitting indicator {}: {}'.format( self.name, v['indicator'], failReason)) self.statistics['error.submit'] += 1 def _push_loop(self): while True: msg = self._queue.get() artifacts = deque() artifacts.append(msg) try: while len(artifacts) < 50: artifacts.append(self._queue.get_nowait()) except Empty: pass while True: retries = 0 try: LOG.info('{} - Sending {} indicators'.format( self.name, len(artifacts))) token = self._get_auth_token() LOG.debug('{} - token: {}'.format(self.name, token)) self._push_indicators(token=token, indicators=artifacts) # Counter already incremented in push_indicators # self.statistics['indicator.tx'] += len(artifacts) break except gevent.GreenletExit: return except HTTPError as e: LOG.error('{} - error submitting indicators - {}'.format( self.name, str(e))) status_code = e.response.status_code if status_code >= 400 and status_code < 500: LOG.error('{}: error in request - {}'.format( self.name, e.response.text)) self.statistics['error.invalid_request'] += 1 break self.statistics['error.submit'] += 1 gevent.sleep(60) except AuthConfigException as e: LOG.exception( '{} - Error submitting indicators - {}'.format( self.name, str(e))) self.statistics['error.submit'] += 1 gevent.sleep(60.0) except WDATPResponseException as e: LOG.exception( '{} - error submitting indicators - {}'.format( self.name, str(e))) self.statistics['error.submit'] += 1 break except Exception as e: LOG.exception( '{} - error submitting indicators - {}'.format( self.name, str(e))) self.statistics['error.submit'] += 1 retries += 1 if retries > 5: break gevent.sleep(120.0) gevent.sleep(0.1) def _encode_indicator(self, indicator, value, expired=False): type_ = MM_2_WDATP_TYPE.get(value['type'], None) if type_ is None: self.statistics['error.unhandled_type'] += 1 raise RuntimeError('{} - Unhandled {}'.format(self.name, type_)) if value['type'] == 'IPv4' and '-' in indicator: a1, a2 = indicator.split('-', 1) r = netaddr.IPRange(a1, a2) indicators = [str(i) for i in r] else: indicators = [indicator] description = '{} indicator from {}'.format( type_, ', '.join(value['sources'])) title = 'MineMeld - {}'.format(indicator) creation = datetime.utcnow() creation = creation.isoformat() + 'Z' expiration = datetime.utcnow() + timedelta(days=365) if expired: expiration = datetime.fromtimestamp(0) expiration = expiration.isoformat( ) + 'Z' # expiration is always in UTC result = [] for i in indicators: d = dict(indicatorValue=i, indicatorType=type_, title=title, description=description, creationTimeDateTimeUtc=creation, expirationTime=expiration, action=self.action) if self.severity is not None: d['severity'] = self.severity result.append(d) return result def _checkpoint_check(self, source=None, value=None): t0 = time.time() while ((time.time() - t0) < 30) and self._queue.qsize() != 0: gevent.sleep(0.5) self._push_glet.kill() LOG.info('{} - checkpoint with {} elements in the queue'.format( self.name, self._queue.qsize())) super(OutputBatch, self).checkpoint(source=source, value=value) @_counting('update.processed') def filtered_update(self, source=None, indicator=None, value=None): try: for i in self._encode_indicator(indicator, value, expired=False): self._queue.put(i, block=True, timeout=0.001) except Full: self.statistics['error.queue_full'] += 1 @_counting('withdraw.processed') def filtered_withdraw(self, source=None, indicator=None, value=None): if value is None: self.statistics['error.no_value'] += 1 return try: for i in self._encode_indicator(indicator, value, expired=True): self._queue.put(i, block=True, timeout=0.001) except Full: self.statistics['error.queue_full'] += 1 @_counting('checkpoint.rx') def checkpoint(self, source=None, value=None): self.state = ft_states.CHECKPOINT self._checkpoint_glet = gevent.spawn(self._checkpoint_check, source, value) def length(self, source=None): return self._queue.qsize() def start(self): super(OutputBatch, self).start() self._push_glet = gevent.spawn(self._push_loop) def stop(self): super(OutputBatch, self).stop() if self._push_glet is not None: self._push_glet.kill() if self._checkpoint_glet is not None: self._checkpoint_glet.kill() self.table.close() def hup(self, source=None): LOG.info('%s - hup received, reload side config', self.name) self._load_side_config() @staticmethod def gc(name, config=None): ActorBaseFT.gc(name, config=config) shutil.rmtree(name, ignore_errors=True)
class Connection(object): in_buffer_size = 4096 out_buffer_size = 4096 cql_version = None protocol_version = 2 keyspace = None compression = True compressor = None decompressor = None ssl_options = None last_error = None in_flight = 0 is_defunct = False is_closed = False lock = None is_control_connection = False def __init__(self, host='127.0.0.1', port=9042, authenticator=None, ssl_options=None, sockopts=None, compression=True, cql_version=None, protocol_version=2, is_control_connection=False): self.host = host self.port = port self.authenticator = authenticator self.ssl_options = ssl_options self.sockopts = sockopts self.compression = compression self.cql_version = cql_version self.protocol_version = protocol_version self.is_control_connection = is_control_connection self._push_watchers = defaultdict(set) self._id_queue = Queue(MAX_STREAM_PER_CONNECTION) for i in range(MAX_STREAM_PER_CONNECTION): self._id_queue.put_nowait(i) self.lock = RLock() @classmethod def initialize_reactor(self): """ Called once by Cluster.connect(). This should be used by implementations to set up any resources that will be shared across connections. """ pass def close(self): raise NotImplementedError() def defunct(self, exc): with self.lock: if self.is_defunct or self.is_closed: return self.is_defunct = True log.debug("Defuncting connection (%s) to %s:", id(self), self.host, exc_info=exc) self.last_error = exc self.close() self.error_all_callbacks(exc) self.connected_event.set() return exc def error_all_callbacks(self, exc): with self.lock: callbacks = self._callbacks self._callbacks = {} new_exc = ConnectionShutdown(str(exc)) for cb in callbacks.values(): try: cb(new_exc) except Exception: log.warning("Ignoring unhandled exception while erroring callbacks for a " "failed connection (%s) to host %s:", id(self), self.host, exc_info=True) def handle_pushed(self, response): log.debug("Message pushed from server: %r", response) for cb in self._push_watchers.get(response.event_type, []): try: cb(response.event_args) except Exception: log.exception("Pushed event handler errored, ignoring:") def send_msg(self, msg, cb, wait_for_id=False): if self.is_defunct: raise ConnectionShutdown("Connection to %s is defunct" % self.host) elif self.is_closed: raise ConnectionShutdown("Connection to %s is closed" % self.host) if not wait_for_id: try: request_id = self._id_queue.get_nowait() except Empty: raise ConnectionBusy( "Connection to %s is at the max number of requests" % self.host) else: request_id = self._id_queue.get() self._callbacks[request_id] = cb self.push(msg.to_binary(request_id, self.protocol_version, compression=self.compressor)) return request_id def wait_for_response(self, msg, timeout=None): return self.wait_for_responses(msg, timeout=timeout)[0] def wait_for_responses(self, *msgs, **kwargs): if self.is_closed or self.is_defunct: raise ConnectionShutdown("Connection %s is already closed" % (self, )) timeout = kwargs.get('timeout') waiter = ResponseWaiter(self, len(msgs)) # busy wait for sufficient space on the connection messages_sent = 0 while True: needed = len(msgs) - messages_sent with self.lock: available = min(needed, MAX_STREAM_PER_CONNECTION - self.in_flight) self.in_flight += available for i in range(messages_sent, messages_sent + available): self.send_msg(msgs[i], partial(waiter.got_response, index=i), wait_for_id=True) messages_sent += available if messages_sent == len(msgs): break else: if timeout is not None: timeout -= 0.01 if timeout <= 0.0: raise OperationTimedOut() time.sleep(0.01) try: return waiter.deliver(timeout) except OperationTimedOut: raise except Exception as exc: self.defunct(exc) raise def register_watcher(self, event_type, callback): raise NotImplementedError() def register_watchers(self, type_callback_dict): raise NotImplementedError() def control_conn_disposed(self): self.is_control_connection = False self._push_watchers = {} @defunct_on_error def process_msg(self, msg, body_len): version, flags, stream_id, opcode = header_unpack(msg[:4]) if stream_id < 0: callback = None else: callback = self._callbacks.pop(stream_id, None) self._id_queue.put_nowait(stream_id) body = None try: # check that the protocol version is supported given_version = version & PROTOCOL_VERSION_MASK if given_version != self.protocol_version: msg = "Server protocol version (%d) does not match the specified driver protocol version (%d). " +\ "Consider setting Cluster.protocol_version to %d." raise ProtocolError(msg % (given_version, self.protocol_version, given_version)) # check that the header direction is correct if version & HEADER_DIRECTION_MASK != HEADER_DIRECTION_TO_CLIENT: raise ProtocolError( "Header direction in response is incorrect; opcode %04x, stream id %r" % (opcode, stream_id)) if body_len > 0: body = msg[8:] elif body_len == 0: body = six.binary_type() else: raise ProtocolError("Got negative body length: %r" % body_len) response = decode_response(stream_id, flags, opcode, body, self.decompressor) except Exception as exc: log.exception("Error decoding response from Cassandra. " "opcode: %04x; message contents: %r", opcode, msg) if callback is not None: callback(exc) self.defunct(exc) return try: if stream_id < 0: self.handle_pushed(response) elif callback is not None: callback(response) except Exception: log.exception("Callback handler errored, ignoring:") @defunct_on_error def _send_options_message(self): if self.cql_version is None and (not self.compression or not locally_supported_compressions): log.debug("Not sending options message for new connection(%s) to %s " "because compression is disabled and a cql version was not " "specified", id(self), self.host) self._compressor = None self.cql_version = DEFAULT_CQL_VERSION self._send_startup_message() else: log.debug("Sending initial options message for new connection (%s) to %s", id(self), self.host) self.send_msg(OptionsMessage(), self._handle_options_response) @defunct_on_error def _handle_options_response(self, options_response): if self.is_defunct: return if not isinstance(options_response, SupportedMessage): if isinstance(options_response, ConnectionException): raise options_response else: log.error("Did not get expected SupportedMessage response; " \ "instead, got: %s", options_response) raise ConnectionException("Did not get expected SupportedMessage " \ "response; instead, got: %s" \ % (options_response,)) log.debug("Received options response on new connection (%s) from %s", id(self), self.host) supported_cql_versions = options_response.cql_versions remote_supported_compressions = options_response.options['COMPRESSION'] if self.cql_version: if self.cql_version not in supported_cql_versions: raise ProtocolError( "cql_version %r is not supported by remote (w/ native " "protocol). Supported versions: %r" % (self.cql_version, supported_cql_versions)) else: self.cql_version = supported_cql_versions[0] self._compressor = None compression_type = None if self.compression: overlap = (set(locally_supported_compressions.keys()) & set(remote_supported_compressions)) if len(overlap) == 0: log.debug("No available compression types supported on both ends." " locally supported: %r. remotely supported: %r", locally_supported_compressions.keys(), remote_supported_compressions) else: compression_type = None if isinstance(self.compression, six.string_types): # the user picked a specific compression type ('snappy' or 'lz4') if self.compression not in remote_supported_compressions: raise ProtocolError( "The requested compression type (%s) is not supported by the Cassandra server at %s" % (self.compression, self.host)) compression_type = self.compression else: # our locally supported compressions are ordered to prefer # lz4, if available for k in locally_supported_compressions.keys(): if k in overlap: compression_type = k break # set the decompressor here, but set the compressor only after # a successful Ready message self._compressor, self.decompressor = \ locally_supported_compressions[compression_type] self._send_startup_message(compression_type) @defunct_on_error def _send_startup_message(self, compression=None): opts = {} if compression: opts['COMPRESSION'] = compression sm = StartupMessage(cqlversion=self.cql_version, options=opts) self.send_msg(sm, cb=self._handle_startup_response) @defunct_on_error def _handle_startup_response(self, startup_response, did_authenticate=False): if self.is_defunct: return if isinstance(startup_response, ReadyMessage): log.debug("Got ReadyMessage on new connection (%s) from %s", id(self), self.host) if self._compressor: self.compressor = self._compressor self.connected_event.set() elif isinstance(startup_response, AuthenticateMessage): log.debug("Got AuthenticateMessage on new connection (%s) from %s: %s", id(self), self.host, startup_response.authenticator) if self.authenticator is None: raise AuthenticationFailed('Remote end requires authentication.') self.authenticator_class = startup_response.authenticator if isinstance(self.authenticator, dict): log.debug("Sending credentials-based auth response on %s", self) cm = CredentialsMessage(creds=self.authenticator) callback = partial(self._handle_startup_response, did_authenticate=True) self.send_msg(cm, cb=callback) else: log.debug("Sending SASL-based auth response on %s", self) initial_response = self.authenticator.initial_response() initial_response = "" if initial_response is None else initial_response.encode('utf-8') self.send_msg(AuthResponseMessage(initial_response), self._handle_auth_response) elif isinstance(startup_response, ErrorMessage): log.debug("Received ErrorMessage on new connection (%s) from %s: %s", id(self), self.host, startup_response.summary_msg()) if did_authenticate: raise AuthenticationFailed( "Failed to authenticate to %s: %s" % (self.host, startup_response.summary_msg())) else: raise ConnectionException( "Failed to initialize new connection to %s: %s" % (self.host, startup_response.summary_msg())) elif isinstance(startup_response, ConnectionShutdown): log.debug("Connection to %s was closed during the startup handshake", (self.host)) raise startup_response else: msg = "Unexpected response during Connection setup: %r" log.error(msg, startup_response) raise ProtocolError(msg % (startup_response,)) @defunct_on_error def _handle_auth_response(self, auth_response): if self.is_defunct: return if isinstance(auth_response, AuthSuccessMessage): log.debug("Connection %s successfully authenticated", self) self.authenticator.on_authentication_success(auth_response.token) if self._compressor: self.compressor = self._compressor self.connected_event.set() elif isinstance(auth_response, AuthChallengeMessage): response = self.authenticator.evaluate_challenge(auth_response.challenge) msg = AuthResponseMessage("" if response is None else response) log.debug("Responding to auth challenge on %s", self) self.send_msg(msg, self._handle_auth_response) elif isinstance(auth_response, ErrorMessage): log.debug("Received ErrorMessage on new connection (%s) from %s: %s", id(self), self.host, auth_response.summary_msg()) raise AuthenticationFailed( "Failed to authenticate to %s: %s" % (self.host, auth_response.summary_msg())) elif isinstance(auth_response, ConnectionShutdown): log.debug("Connection to %s was closed during the authentication process", self.host) raise auth_response else: msg = "Unexpected response during Connection authentication to %s: %r" log.error(msg, self.host, auth_response) raise ProtocolError(msg % (self.host, auth_response)) def set_keyspace_blocking(self, keyspace): if not keyspace or keyspace == self.keyspace: return query = QueryMessage(query='USE "%s"' % (keyspace,), consistency_level=ConsistencyLevel.ONE) try: result = self.wait_for_response(query) except InvalidRequestException as ire: # the keyspace probably doesn't exist raise ire.to_exception() except Exception as exc: conn_exc = ConnectionException( "Problem while setting keyspace: %r" % (exc,), self.host) self.defunct(conn_exc) raise conn_exc if isinstance(result, ResultMessage): self.keyspace = keyspace else: conn_exc = ConnectionException( "Problem while setting keyspace: %r" % (result,), self.host) self.defunct(conn_exc) raise conn_exc def set_keyspace_async(self, keyspace, callback): """ Use this in order to avoid deadlocking the event loop thread. When the operation completes, `callback` will be called with two arguments: this connection and an Exception if an error occurred, otherwise :const:`None`. """ if not keyspace or keyspace == self.keyspace: callback(self, None) return query = QueryMessage(query='USE "%s"' % (keyspace,), consistency_level=ConsistencyLevel.ONE) def process_result(result): if isinstance(result, ResultMessage): self.keyspace = keyspace callback(self, None) elif isinstance(result, InvalidRequestException): callback(self, result.to_exception()) else: callback(self, self.defunct(ConnectionException( "Problem while setting keyspace: %r" % (result,), self.host))) self.send_msg(query, process_result, wait_for_id=True) def __str__(self): status = "" if self.is_defunct: status = " (defunct)" elif self.is_closed: status = " (closed)" return "<%s(%r) %s:%d%s>" % (self.__class__.__name__, id(self), self.host, self.port, status) __repr__ = __str__
class Output(ActorBaseFT): def __init__(self, name, chassis, config): self._queue = None super(Output, self).__init__(name, chassis, config) self._push_glet = None self._checkpoint_glet = None self.api_client_id = str(uuid.uuid4()) self.sequence_number = 0 def configure(self): super(Output, self).configure() self.queue_maxsize = int(self.config.get('queue_maxsize', 100000)) if self.queue_maxsize == 0: self.queue_maxsize = None self._queue = Queue(maxsize=self.queue_maxsize) self.client_id = self.config.get('client_id', None) self.client_secret = self.config.get('client_secret', None) self.tenant_id = self.config.get('tenant_id', None) self.sender_id = self.config.get('sender_id', 'minemeld') self.side_config_path = self.config.get('side_config', None) if self.side_config_path is None: self.side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '%s_side_config.yml' % self.name) self._load_side_config() def _load_side_config(self): try: with open(self.side_config_path, 'r') as f: sconfig = yaml.safe_load(f) except Exception as e: LOG.error('%s - Error loading side config: %s', self.name, str(e)) return client_id = sconfig.get('client_id', None) if client_id is not None: self.client_id = client_id LOG.info('{} - client_id set'.format(self.name)) client_secret = sconfig.get('client_secret', None) if client_secret is not None: self.client_secret = client_secret LOG.info('{} - client_secret set'.format(self.name)) tenant_id = sconfig.get('tenant_id', None) if tenant_id is not None: self.tenant_id = tenant_id LOG.info('{} - tenant_id set'.format(self.name)) def _saved_state_restore(self, saved_state): super(Output, self)._saved_state_restore(saved_state) self.api_client_id = saved_state.get('api_client_id', None) self.sequence_number = saved_state.get('sequence_number', None) LOG.info( '{} - saved state: api_client_id: {} sequence_number: {}'.format( self.name, self.api_client_id, self.sequence_number)) def _saved_state_create(self): sstate = super(Output, self)._saved_state_create() sstate['api_client_id'] = self.api_client_id sstate['sequence_number'] = self.sequence_number return sstate def _saved_state_reset(self): super(Output, self)._saved_state_reset() self.api_client_id = str(uuid.uuid4()) self.sequence_number = 0 def connect(self, inputs, output): output = False super(Output, self).connect(inputs, output) def initialize(self): pass def rebuild(self): pass def reset(self): pass def _get_auth_token(self): if self.client_id is None: LOG.error('{} - client_id not set'.format(self.name)) raise AuthConfigException('{} - client_id not set'.format( self.name)) if self.client_secret is None: LOG.error('{} - client_secret not set'.format(self.name)) raise AuthConfigException('{} - client_secret not set'.format( self.name)) if self.tenant_id is None: LOG.error('{} - tenant_id not set'.format(self.name)) raise AuthConfigException('{} - tenant_id not set'.format( self.name)) context = adal.AuthenticationContext( AUTHORITY_URL.format(self.tenant_id), validate_authority=self.tenant_id != 'adfs', api_version=None) token = context.acquire_token_with_client_credentials( RESOURCE, self.client_id, self.client_secret) if token is None or 'accessToken' not in token: LOG.error('{} - Invalid token or accessToken not available'.format( self.name)) raise RuntimeError( '{} - Invalid token or accessToken not available'.format( self.name)) return token['accessToken'] def _get_endpoint_orgid(self, token): # this should look like # { # u'AadTenantId': u'bb19bb5c-0e8d-4a73-bd6b-d015b298ecd7', # u'ServiceUri': u'https://partnerstifrontend-eus-prd.trafficmanager.net/threatintel/indicators', # u'ServiceType': 1, # u'WdAtpOrgId': u'55c01df7-a1eb-4eae-ae3b-a9b423d07d72' # } result = requests.get(ENDPOINT_URL, headers={ 'Authorization': 'Bearer {}'.format(token), 'Content-Type': 'application/json' }) result.raise_for_status() result = result.json() LOG.debug('{} - endpoints: {}'.format(self.name, result)) if result.get('AadTenantId', None) != self.tenant_id: raise AuthConfigException( '{} - Endpoint response AadTenantId differs from tenant_id: {}' .format(self.name, result)) endpoint = result.get('ServiceUri', None) if endpoint is None: raise AuthConfigException( '{} - Endpoint response missing ServiceUri field: {}'.format( self.name, result)) org_id = result.get('WdAtpOrgId', None) if endpoint is None: raise AuthConfigException( '{} - Endpoint response missing WdAtpOrgId field: {}'.format( self.name, result)) return endpoint, org_id def _push_indicators(self, token, endpoint, org_id, indicators): # DEPRECATED # message = { # 'Id': self.api_client_id, # 'SequenceNumber': self.sequence_number, # 'SenderId': self.sender_id, # 'Indicators': list(indicators), # 'WdAtpOrgId': org_id # } # LOG.debug(message) # result = requests.post( # endpoint, # headers={ # 'Content-Type': 'application/json', # 'Authorization': 'Bearer {}'.format(token) # }, # json=message # ) # LOG.debug(result.text) # result.raise_for_status() raise WDATPResponseException( 'This output node is deprecated: please switch to OutputBatch') def _push_loop(self): while True: msg = self._queue.get() artifacts = deque() artifacts.append(msg) try: while len(artifacts) < 511: artifacts.append(self._queue.get_nowait()) except Empty: pass while True: result = None try: LOG.info('{} - Sending {}:{}'.format( self.name, self.api_client_id, self.sequence_number)) # DEPRECATED - no need to get the token # token = self._get_auth_token() token = 'DEPRECATED' LOG.debug('{} - token: {}'.format(self.name, token)) # DEPRECATED #endpoint, org_id = self._get_endpoint_orgid(token) #LOG.debug('{} - endpoint: {} WdAtpOrgId: {}'.format(self.name, endpoint, org_id)) # self._push_indicators( # token=token, # endpoint=endpoint, # org_id=org_id, # indicators=artifacts # ) self._push_indicators(None, None, None, None) self.sequence_number += 1 self.statistics['indicator.tx'] += len(artifacts) break except gevent.GreenletExit: return except RequestException as e: LOG.error('{} - error submitting indicators - {}'.format( self.name, str(e))) if result is not None and result.status_code >= 400 and result.status_code < 500: LOG.error('{}: error in request - {}'.format( self.name, result.text)) self.statistics['error.invalid_request'] += 1 break self.statistics['error.submit'] += 1 gevent.sleep(60) except AuthConfigException as e: LOG.exception( '{} - Error submitting indicators - {}'.format( self.name, str(e))) self.statistics['error.submit'] += 1 gevent.sleep(60.0) except WDATPResponseException as e: LOG.exception( '{} - error submitting indicators - {}'.format( self.name, str(e))) self.statistics['error.submit'] += 1 break except Exception as e: LOG.exception( '{} - error submitting indicators - {}'.format( self.name, str(e))) self.statistics['error.submit'] += 1 gevent.sleep(120.0) gevent.sleep(0.1) def _encode_indicator(self, indicator, value, expired=False): type_ = value['type'] if type_ not in ['URL', 'domain', 'md5', 'sha256', 'IPv4']: self.statistics['error.unhandled_type'] += 1 raise RuntimeError('{} - Unhandled {}'.format(self.name, type_)) description = '{} indicator from {}'.format( type_, ', '.join(value['sources'])) external_id = '{}:{}'.format(type_, indicator) expiration = datetime.utcnow() + timedelta(days=365) if expired: expiration = datetime.fromtimestamp(0) expiration = expiration.isoformat() indicators = [] if type_ == 'IPv4' and '-' in indicator: # October 2020 # as MSFT removed the support for CIDRs in the API # we must translate the ranges in individual IPv4 indicators a1, a2 = indicator.split('-', 1) r = netaddr.IPRange(a1, a2) indicators = [str(i) for i in r] else: indicators = [indicator] result = [] for i in indicators: r = { 'Description': description, 'Confidence': value['confidence'], 'ExternalId': external_id, 'IndicatorExpirationDateTime': expiration } if type_ == 'URL': r['Url'] = indicator elif type_ == 'domain': r['DNSDomainName'] = indicator elif type_ == 'md5': r['FileMD5'] = indicator elif type_ == 'sha256': r['FileSha256'] = indicator elif type_ == 'IPv4': r['NetworkDestinationIPv4'] = str(indicator) else: # Unsupported indicator type, should never reach this code continue LOG.debug('{!r} - add indicator {!r} to queue'.format( self.name, r)) result.append(r) return result def _checkpoint_check(self, source=None, value=None): t0 = time.time() while ((time.time() - t0) < 30) and self._queue.qsize() != 0: gevent.sleep(0.5) self._push_glet.kill() LOG.info('{} - checkpoint with {} elements in the queue'.format( self.name, self._queue.qsize())) super(Output, self).checkpoint(source=source, value=value) @_counting('update.processed') def filtered_update(self, source=None, indicator=None, value=None): try: for i in self._encode_indicator(indicator, value, expired=False): self._queue.put(i, block=True, timeout=0.001) except Full: self.statistics['error.queue_full'] += 1 @_counting('withdraw.processed') def filtered_withdraw(self, source=None, indicator=None, value=None): if value is None: self.statistics['error.no_value'] += 1 return try: for i in self._encode_indicator(indicator, value, expired=True): self._queue.put(i, block=True, timeout=0.001) except Full: self.statistics['error.queue_full'] += 1 @_counting('checkpoint.rx') def checkpoint(self, source=None, value=None): self.state = ft_states.CHECKPOINT self._checkpoint_glet = gevent.spawn(self._checkpoint_check, source, value) def mgmtbus_status(self): result = super(ActorBaseFT, self).mgmtbus_status() result['sub_state'] = 'ERROR' result['sub_state_message'] = 'This node is deprecated' return result def length(self, source=None): return self._queue.qsize() def start(self): super(Output, self).start() self._push_glet = gevent.spawn(self._push_loop) def stop(self): super(Output, self).stop() if self._push_glet is not None: self._push_glet.kill() if self._checkpoint_glet is not None: self._checkpoint_glet.kill() def hup(self, source=None): LOG.info('%s - hup received, reload side config', self.name) self._load_side_config() @staticmethod def gc(name, config=None): ActorBaseFT.gc(name, config=config) shutil.rmtree(name, ignore_errors=True)