class gather(Stream):
def __init__(self, child, limit=10, client=None):
self.client = client or default_client()
self.queue = Queue(maxsize=limit)
self.condition = Condition()
Stream.__init__(self, child)
self.client.loop.add_callback(self.cb)
def update(self, x, who=None):
return self.queue.put(x)
@gen.coroutine
def cb(self):
while True:
x = yield self.queue.get()
L = [x]
while not self.queue.empty():
L.append(self.queue.get_nowait())
results = yield self.client._gather(L)
for x in results:
yield self.emit(x)
if self.queue.empty():
self.condition.notify_all()
@gen.coroutine
def flush(self):
while not self.queue.empty():
yield self.condition.wait()
async def test_reader(message, repeats, interval, add_excess,
communicator_spawner):
queue = Queue()
process = communicator_spawner.spawn_writer(message=message,
repeats=repeats,
interval=interval,
add_excess=add_excess)
reader = LspStdIoReader(stream=process.stdout, queue=queue)
await asyncio.gather(join_process(process, headstart=3, timeout=1),
reader.read())
result = queue.get_nowait()
assert result == message * repeats
class BatchedStream(object):
""" Mostly obsolete, see BatchedSend """
def __init__(self, stream, interval):
self.stream = stream
self.interval = interval / 1000.0
self.last_transmission = default_timer()
self.send_q = Queue()
self.recv_q = Queue()
self._background_send_coroutine = self._background_send()
self._background_recv_coroutine = self._background_recv()
self._broken = None
self.pc = PeriodicCallback(lambda: None, 100)
self.pc.start()
@gen.coroutine
def _background_send(self):
with log_errors():
while True:
msg = yield self.send_q.get()
if msg == "close":
break
msgs = [msg]
now = default_timer()
wait_time = self.last_transmission + self.interval - now
if wait_time > 0:
yield gen.sleep(wait_time)
while not self.send_q.empty():
msgs.append(self.send_q.get_nowait())
try:
yield write(self.stream, msgs)
except StreamClosedError:
self.recv_q.put_nowait("close")
self._broken = True
break
if len(msgs) > 1:
logger.debug("Batched messages: %d", len(msgs))
for _ in msgs:
self.send_q.task_done()
@gen.coroutine
def _background_recv(self):
with log_errors():
while True:
try:
msgs = yield read(self.stream)
except StreamClosedError:
self.recv_q.put_nowait("close")
self.send_q.put_nowait("close")
self._broken = True
break
assert isinstance(msgs, list)
if len(msgs) > 1:
logger.debug("Batched messages: %d", len(msgs))
for msg in msgs:
self.recv_q.put_nowait(msg)
@gen.coroutine
def flush(self):
yield self.send_q.join()
@gen.coroutine
def send(self, msg):
if self._broken:
raise StreamClosedError("Batch Stream is Closed")
else:
self.send_q.put_nowait(msg)
@gen.coroutine
def recv(self):
result = yield self.recv_q.get()
if result == "close":
raise StreamClosedError("Batched Stream is Closed")
else:
raise gen.Return(result)
@gen.coroutine
def close(self):
yield self.flush()
raise gen.Return(self.stream.close())
def closed(self):
return self.stream.closed()
class SQSDrain(object):
"""Implementation of IDrain that writes to an AWS SQS queue.
"""
def __init__(self, logger, loop, sqs_client,
metric_prefix='emitter'):
self.emitter = sqs_client
self.logger = logger
self.loop = loop
self.metric_prefix = metric_prefix
self.output_error = Event()
self.state = RUNNING
self.sender_tag = 'sender:%s.%s' % (self.__class__.__module__,
self.__class__.__name__)
self._send_queue = Queue()
self._should_flush_queue = Event()
self._flush_handle = None
self.loop.spawn_callback(self._onSend)
@gen.coroutine
def _flush_send_batch(self, batch_size):
send_batch = [
self._send_queue.get_nowait()
for pos in range(min(batch_size, self.emitter.max_messages))
]
try:
response = yield self.emitter.send_message_batch(*send_batch)
except SQSError as err:
self.logger.exception('Error encountered flushing data to SQS: %s',
err)
self.output_error.set()
for msg in send_batch:
self._send_queue.put_nowait(msg)
else:
if response.Failed:
self.output_error.set()
for req in response.Failed:
self.logger.error('Message failed to send: %s', req.Id)
self._send_queue.put_nowait(req)
@gen.coroutine
def _onSend(self):
respawn = True
while respawn:
qsize = self._send_queue.qsize()
# This will keep flushing until clear,
# including items that show up in between flushes
while qsize > 0:
try:
yield self._flush_send_batch(qsize)
except Exception as err:
self.logger.exception(err)
self.output_error.set()
qsize = self._send_queue.qsize()
# We've cleared the backlog, remove any possible future flush
if self._flush_handle:
self.loop.remove_timeout(self._flush_handle)
self._flush_handle = None
self._should_flush_queue.clear()
yield self._should_flush_queue.wait()
@gen.coroutine
def close(self, timeout=None):
self.state = CLOSING
yield self._send_queue.join(timeout)
def emit_nowait(self, msg):
if self._send_queue.qsize() >= self.emitter.max_messages:
# Signal flush
self._should_flush_queue.set()
raise QueueFull()
elif self._flush_handle is None:
# Ensure we flush messages at least by MAX_TIMEOUT
self._flush_handle = self.loop.add_timeout(
MAX_TIMEOUT,
lambda: self._should_flush_queue.set(),
)
self.logger.debug("Drain emitting")
self._send_queue.put_nowait(msg)
@gen.coroutine
def emit(self, msg, timeout=None):
if self._send_queue.qsize() >= self.emitter.max_messages:
# Signal flush
self._should_flush_queue.set()
elif self._flush_handle is None:
# Ensure we flush messages at least by MAX_TIMEOUT
self._flush_handle = self.loop.add_timeout(
MAX_TIMEOUT,
lambda: self._should_flush_queue.set(),
)
yield self._send_queue.put(msg, timeout)
class Model:
def __init__(self, config_file):
self.lock = locks.Lock()
self.classification_queue = Queue()
print('loading config %s' % config_file, file=log.v5)
# Load and setup config
try:
self.config = Config.Config()
self.config.load_file(config_file)
self.pause_after_first_seq = self.config.float(
'pause_after_first_seq', 0.2)
self.batch_size = self.config.int('batch_size', 5000)
self.max_seqs = self.config.int('max_seqs', -1)
except Exception:
print('Error: loading config %s failed' % config_file, file=log.v1)
raise
try:
self.devices = self._init_devices()
except Exception:
print('Error: Loading devices for config %s failed' % config_file,
file=log.v1)
raise
print('Starting engine for config %s' % config_file, file=log.v5)
self.engine = Engine.Engine(self.devices)
try:
self.engine.init_network_from_config(config=self.config)
except Exception:
print('Error: Loading network for config %s failed' % config_file,
file=log.v1)
raise
IOLoop.current().spawn_callback(self.classify_in_background)
self.last_used = datetime.datetime.now()
def _init_devices(self):
"""
Initiates the required devices for a config. Same as the funtion initDevices in
rnn.py.
:param config:
:return: A list with the devices used.
"""
oldDeviceConfig = ",".join(self.config.list('device', ['default']))
if "device" in TheanoFlags:
# This is important because Theano likely already has initialized that device.
config.set("device", TheanoFlags["device"])
print("Devices: Use %s via THEANO_FLAGS instead of %s." %
(TheanoFlags["device"], oldDeviceConfig),
file=log.v4)
devArgs = getDevicesInitArgs(self.config)
assert len(devArgs) > 0
devices = [Device(**kwargs) for kwargs in devArgs]
for device in devices:
while not device.initialized:
time.sleep(0.25)
if devices[0].blocking:
print("Devices: Used in blocking / single proc mode.", file=log.v4)
else:
print("Devices: Used in multiprocessing mode.", file=log.v4)
return devices
@tornado.gen.coroutine
def classify_in_background(self):
while True:
requests = []
# fetch first request
r = yield self.classification_queue.get()
requests.append(r)
# grab all other waiting requests
try:
while True:
requests.append(self.classification_queue.get_nowait())
except QueueEmpty:
pass
output_dim = {}
# Do dataset creation and classification.
dataset = StaticDataset(data=[r.data for r in requests],
output_dim=output_dim)
dataset.init_seq_order()
batches = dataset.generate_batches(
recurrent_net=self.engine.network.recurrent,
batch_size=self.batch_size,
max_seqs=self.max_seqs)
with (yield self.lock.acquire()):
ctt = ForwardTaskThread(self.engine.network, self.devices,
dataset, batches)
yield ctt.join()
try:
for i in range(dataset.num_seqs):
requests[i].future.set_result(ctt.result[i])
self.classification_queue.task_done()
except Exception as e:
print('exception', e)
raise
@tornado.gen.coroutine
def classify(self, data):
self.last_used = datetime.datetime.now()
request = ClassificationRequest(data)
yield self.classification_queue.put(request)
yield request.future
return request.future.result()
class BatchedStream(object):
""" Mostly obsolete, see BatchedSend """
def __init__(self, stream, interval):
self.stream = stream
self.interval = interval / 1000.
self.last_transmission = default_timer()
self.send_q = Queue()
self.recv_q = Queue()
self._background_send_coroutine = self._background_send()
self._background_recv_coroutine = self._background_recv()
self._broken = None
self.pc = PeriodicCallback(lambda: None, 100)
self.pc.start()
@gen.coroutine
def _background_send(self):
with log_errors():
while True:
msg = yield self.send_q.get()
if msg == 'close':
break
msgs = [msg]
now = default_timer()
wait_time = self.last_transmission + self.interval - now
if wait_time > 0:
yield gen.sleep(wait_time)
while not self.send_q.empty():
msgs.append(self.send_q.get_nowait())
try:
yield write(self.stream, msgs)
except StreamClosedError:
self.recv_q.put_nowait('close')
self._broken = True
break
if len(msgs) > 1:
logger.debug("Batched messages: %d", len(msgs))
for _ in msgs:
self.send_q.task_done()
@gen.coroutine
def _background_recv(self):
with log_errors():
while True:
try:
msgs = yield read(self.stream)
except StreamClosedError:
self.recv_q.put_nowait('close')
self.send_q.put_nowait('close')
self._broken = True
break
assert isinstance(msgs, list)
if len(msgs) > 1:
logger.debug("Batched messages: %d", len(msgs))
for msg in msgs:
self.recv_q.put_nowait(msg)
@gen.coroutine
def flush(self):
yield self.send_q.join()
@gen.coroutine
def send(self, msg):
if self._broken:
raise StreamClosedError('Batch Stream is Closed')
else:
self.send_q.put_nowait(msg)
@gen.coroutine
def recv(self):
result = yield self.recv_q.get()
if result == 'close':
raise StreamClosedError('Batched Stream is Closed')
else:
raise gen.Return(result)
@gen.coroutine
def close(self):
yield self.flush()
raise gen.Return(self.stream.close())
def closed(self):
return self.stream.closed()
class ChannelConfiguration:
_USER_CLOSE_CODE = 0
_NORMAL_CLOSE_CODE = 200
_NO_ROUTE_CODE = 312
logger = logger
connection: AsyncConnection
def __init__(self, connection: AsyncConnection, io_loop, exchange=None, exchange_type=None, queue=None,
routing_key=None, durable=False, auto_delete=False, prefetch_count=None):
self._channel = None
self.connection = connection
self._io_loop = io_loop
self._channel_queue = Queue(maxsize=1)
self._queue = queue if queue is not None else ""
self._exchange = exchange
if exchange_type is None:
exchange_type = 'topic'
self._exchange_type = exchange_type
self._routing_key = routing_key
self._durable = durable
self._auto_delete = auto_delete
if prefetch_count is None:
prefetch_count = 1
self._prefetch_count = prefetch_count
self._should_consume = False
self._consume_params = dict()
@gen.coroutine
def consume(self, on_message_callback, handler=None, no_ack=False):
self.logger.info(f"[start consuming] routing key: {self._routing_key}; queue name: {self._queue}")
channel = yield self._get_channel()
self._should_consume = True
self._consume_params = [on_message_callback, handler, no_ack]
if handler is not None:
channel.basic_consume(
queue=self._queue,
auto_ack=no_ack,
on_message_callback=functools.partial(
on_message_callback,
handler=handler
)
)
else:
channel.basic_consume(queue=self._queue, on_message_callback=on_message_callback, auto_ack=no_ack)
@gen.coroutine
def publish(self, body, mandatory=None, properties=None, reply_to=None):
channel = yield self._get_channel()
if reply_to is not None:
exchange = ""
routing_key = reply_to
else:
exchange = self._exchange
routing_key = self._routing_key
self.logger.info(f"Publishing message. exchange: {exchange}; routing_key: {routing_key}")
channel.basic_publish(exchange=exchange, routing_key=routing_key, body=body,
mandatory=mandatory, properties=properties)
@gen.coroutine
def _get_channel(self):
if self._channel_queue.empty():
yield self._create_channel()
channel = yield self._top()
return channel
@gen.coroutine
def _top(self):
channel = yield self._channel_queue.get()
self._channel_queue.put(channel)
return channel
def _remove_channel_from_queue(self):
try:
self._channel_queue.get_nowait()
except QueueEmpty:
pass
@gen.coroutine
def _create_channel(self):
self.logger.info("creating channel")
connection = yield self.connection.get_connection()
def on_channel_flow(*args, **kwargs):
pass
def on_channel_cancel(frame):
self.logger.error("Channel was canceled")
if not self._channel_queue.empty():
channel = self._channel
if channel and not channel.is_close or channel.is_closing:
channel.close()
def on_channel_closed(channel, reason):
reply_code, reply_txt = reason.args
self.logger.info(f'Channel {channel} was closed: {reason}')
if reply_code not in [self._NORMAL_CLOSE_CODE, self._USER_CLOSE_CODE]:
self.logger.error(f"Channel closed. reply code: {reply_code}; reply text: {reply_txt}. "
f"System will exist")
if connection and not (connection.is_closed or connection.is_closing):
connection.close()
self._remove_channel_from_queue()
self._io_loop.call_later(1, self._create_channel)
else:
self.logger.info(f"Reply code: {reply_code}, reply text: {reply_txt}")
def on_channel_return(channel, method, property, body):
""""If publish message has failed, this method will be invoked."""
self.logger.error(f"Rejected from server. reply code: {method.reply_code}, reply text: {method.reply_txt}")
raise Exception("Failed to publish message.")
def open_callback(channel):
self.logger.info("Created channel")
channel.add_on_close_callback(on_channel_closed)
channel.add_on_return_callback(on_channel_return)
channel.add_on_flow_callback(on_channel_flow)
channel.add_on_cancel_callback(on_channel_cancel)
self._channel = channel
if self._exchange is not None:
self._exchange_declare()
else:
self._queue_declare()
connection.channel(on_open_callback=open_callback)
def _exchange_declare(self):
self.logger.info(f"Declaring exchange: {self._exchange}")
self._channel.exchange_declare(
callback=self._on_exchange_declared,
exchange=self._exchange,
exchange_type=self._exchange_type,
durable=self._durable,
auto_delete=self._auto_delete)
def _on_exchange_declared(self, unframe):
self.logger.info(f"Declared exchange: {self._exchange}")
self._queue_declare()
def _queue_declare(self):
self.logger.info(f"Declaring queue: {self._queue}")
self._channel.queue_declare(
callback=self._on_queue_declared, queue=self._queue, durable=self._durable, auto_delete=self._auto_delete)
def _on_queue_declared(self, method_frame):
self.logger.info(f"Declared queue: {method_frame.method.queue}")
self._queue = method_frame.method.queue
if self._exchange is not None:
self._queue_bind()
else:
self._on_setup_complete()
def _queue_bind(self):
self.logger.info(f"Binding queue: {self._queue} to exchange: {self._exchange}")
self._channel.queue_bind(
callback=self._on_queue_bind_ok, queue=self._queue, exchange=self._exchange, routing_key=self._routing_key)
def _on_queue_bind_ok(self, unframe):
self.logger.info(f"bound queue: {self._queue} to exchange: {self._exchange}")
self._on_setup_complete()
def _on_setup_complete(self):
self._channel.basic_qos(prefetch_count=self._prefetch_count)
self._channel_queue.put(self._channel)
if self._should_consume:
self._io_loop.call_later(0.01, self.consume, *self._consume_params)
class AsyncTaskManager(object):
"""
Aucote uses asynchronous task executed in ioloop. Some of them,
especially scanners, should finish before ioloop will stop
This class should be accessed by instance class method, which returns global instance of task manager
"""
_instances = {}
TASKS_POLITIC_WAIT = 0
TASKS_POLITIC_KILL_WORKING_FIRST = 1
TASKS_POLITIC_KILL_PROPORTIONS = 2
TASKS_POLITIC_KILL_WORKING = 3
def __init__(self, parallel_tasks=10):
self._shutdown_condition = Event()
self._stop_condition = Event()
self._cron_tasks = {}
self._parallel_tasks = parallel_tasks
self._tasks = Queue()
self._task_workers = {}
self._events = {}
self._limit = self._parallel_tasks
self._next_task_number = 0
self._toucan_keys = {}
@classmethod
def instance(cls, name=None, **kwargs):
"""
Return instance of AsyncTaskManager
Returns:
AsyncTaskManager
"""
if cls._instances.get(name) is None:
cls._instances[name] = AsyncTaskManager(**kwargs)
return cls._instances[name]
@property
def shutdown_condition(self):
"""
Event which is resolved if every job is done and AsyncTaskManager is ready to shutdown
Returns:
Event
"""
return self._shutdown_condition
def start(self):
"""
Start CronTabCallback tasks
Returns:
None
"""
for task in self._cron_tasks.values():
task.start()
for number in range(self._parallel_tasks):
self._task_workers[number] = IOLoop.current().add_callback(
partial(self.process_tasks, number))
self._next_task_number = self._parallel_tasks
def add_crontab_task(self, task, cron, event=None):
"""
Add function to scheduler and execute at cron time
Args:
task (function):
cron (str): crontab value
event (Event): event which prevent from running task with similar aim, eg. security scans
Returns:
None
"""
if event is not None:
event = self._events.setdefault(event, Event())
self._cron_tasks[task] = AsyncCrontabTask(cron, task, event)
@gen.coroutine
def stop(self):
"""
Stop CronTabCallback tasks and wait on them to finish
Returns:
None
"""
for task in self._cron_tasks.values():
task.stop()
IOLoop.current().add_callback(self._prepare_shutdown)
yield [self._stop_condition.wait(), self._tasks.join()]
self._shutdown_condition.set()
def _prepare_shutdown(self):
"""
Check if ioloop can be stopped
Returns:
None
"""
if any(task.is_running() for task in self._cron_tasks.values()):
IOLoop.current().add_callback(self._prepare_shutdown)
return
self._stop_condition.set()
def clear(self):
"""
Clear list of tasks
Returns:
None
"""
self._cron_tasks = {}
self._shutdown_condition.clear()
self._stop_condition.clear()
async def process_tasks(self, number):
"""
Execute queue. Every task in executed in separated thread (_Executor)
"""
log.info("Starting worker %s", number)
while True:
try:
item = self._tasks.get_nowait()
try:
log.debug("Worker %s: starting %s", number, item)
thread = _Executor(task=item, number=number)
self._task_workers[number] = thread
thread.start()
while thread.is_alive():
await sleep(0.5)
except:
log.exception("Worker %s: exception occurred", number)
finally:
log.debug("Worker %s: %s finished", number, item)
self._tasks.task_done()
tasks_per_scan = (
'{}: {}'.format(scanner, len(tasks))
for scanner, tasks in self.tasks_by_scan.items())
log.debug("Tasks left in queue: %s (%s)",
self.unfinished_tasks, ', '.join(tasks_per_scan))
self._task_workers[number] = None
except QueueEmpty:
await gen.sleep(0.5)
if self._stop_condition.is_set() and self._tasks.empty():
return
finally:
if self._limit < len(self._task_workers):
break
del self._task_workers[number]
log.info("Closing worker %s", number)
def add_task(self, task):
"""
Add task to the queue
Args:
task:
Returns:
None
"""
self._tasks.put(task)
@property
def unfinished_tasks(self):
"""
Task which are still processed or in queue
Returns:
int
"""
return self._tasks._unfinished_tasks
@property
def tasks_by_scan(self):
"""
Returns queued tasks grouped by scan
"""
tasks = self._tasks._queue
return_value = {}
for task in tasks:
return_value.setdefault(task.context.scanner.NAME, []).append(task)
return return_value
@property
def cron_tasks(self):
"""
List of cron tasks
Returns:
list
"""
return self._cron_tasks.values()
def cron_task(self, name):
for task in self._cron_tasks.values():
if task.func.NAME == name:
return task
def change_throttling_toucan(self, key, value):
self.change_throttling(value)
def change_throttling(self, new_value):
"""
Change throttling value. Keeps throttling value between 0 and 1.
Behaviour of algorithm is described in docs/throttling.md
Only working tasks are closing here. Idle workers are stop by themselves
"""
if new_value > 1:
new_value = 1
if new_value < 0:
new_value = 0
new_value = round(new_value * 100) / 100
old_limit = self._limit
self._limit = round(self._parallel_tasks * float(new_value))
working_tasks = [
number for number, task in self._task_workers.items()
if task is not None
]
current_tasks = len(self._task_workers)
task_politic = cfg['service.scans.task_politic']
if task_politic == self.TASKS_POLITIC_KILL_WORKING_FIRST:
tasks_to_kill = current_tasks - self._limit
elif task_politic == self.TASKS_POLITIC_KILL_PROPORTIONS:
tasks_to_kill = round((old_limit - self._limit) *
len(working_tasks) / self._parallel_tasks)
elif task_politic == self.TASKS_POLITIC_KILL_WORKING:
tasks_to_kill = (old_limit - self._limit) - (
len(self._task_workers) - len(working_tasks))
else:
tasks_to_kill = 0
log.debug('%s tasks will be killed', tasks_to_kill)
for number in working_tasks:
if tasks_to_kill <= 0:
break
self._task_workers[number].stop()
tasks_to_kill -= 1
self._limit = round(self._parallel_tasks * float(new_value))
current_tasks = len(self._task_workers)
for number in range(self._limit - current_tasks):
self._task_workers[self._next_task_number] = None
IOLoop.current().add_callback(
partial(self.process_tasks, self._next_task_number))
self._next_task_number += 1
class Model:
def __init__(self, config_file):
self.lock = locks.Lock()
self.classification_queue = Queue()
print('loading config %s' % config_file, file=log.v5)
# Load and setup config
try:
self.config = Config.Config()
self.config.load_file(config_file)
self.pause_after_first_seq = self.config.float('pause_after_first_seq', 0.2)
self.batch_size = self.config.int('batch_size', 5000)
self.max_seqs = self.config.int('max_seqs', -1)
except Exception:
print('Error: loading config %s failed' % config_file, file=log.v1)
raise
try:
self.devices = self._init_devices()
except Exception:
print('Error: Loading devices for config %s failed' % config_file, file=log.v1)
raise
print('Starting engine for config %s' % config_file, file=log.v5)
self.engine = Engine.Engine(self.devices)
try:
self.engine.init_network_from_config(config=self.config)
except Exception:
print('Error: Loading network for config %s failed' % config_file, file=log.v1)
raise
IOLoop.current().spawn_callback(self.classify_in_background)
self.last_used = datetime.datetime.now()
def _init_devices(self):
"""
Initiates the required devices for a config. Same as the funtion initDevices in
rnn.py.
:param config:
:return: A list with the devices used.
"""
oldDeviceConfig = ",".join(self.config.list('device', ['default']))
if "device" in TheanoFlags:
# This is important because Theano likely already has initialized that device.
config.set("device", TheanoFlags["device"])
print("Devices: Use %s via THEANO_FLAGS instead of %s." % (TheanoFlags["device"], oldDeviceConfig), file=log.v4)
devArgs = get_devices_init_args(self.config)
assert len(devArgs) > 0
devices = [Device(**kwargs) for kwargs in devArgs]
for device in devices:
while not device.initialized:
time.sleep(0.25)
if devices[0].blocking:
print("Devices: Used in blocking / single proc mode.", file=log.v4)
else:
print("Devices: Used in multiprocessing mode.", file=log.v4)
return devices
@tornado.gen.coroutine
def classify_in_background(self):
while True:
requests = []
# fetch first request
r = yield self.classification_queue.get()
requests.append(r)
# grab all other waiting requests
try:
while True:
requests.append(self.classification_queue.get_nowait())
except QueueEmpty:
pass
output_dim = {}
# Do dataset creation and classification.
dataset = StaticDataset(data=[r.data for r in requests], output_dim=output_dim)
dataset.init_seq_order()
batches = dataset.generate_batches(recurrent_net=self.engine.network.recurrent,
batch_size=self.batch_size, max_seqs=self.max_seqs)
with (yield self.lock.acquire()):
ctt = ForwardTaskThread(self.engine.network, self.devices, dataset, batches)
yield ctt.join()
try:
for i in range(dataset.num_seqs):
requests[i].future.set_result(ctt.result[i])
self.classification_queue.task_done()
except Exception as e:
print('exception', e)
raise
@tornado.gen.coroutine
def classify(self, data):
self.last_used = datetime.datetime.now()
request = ClassificationRequest(data)
yield self.classification_queue.put(request)
yield request.future
return request.future.result()
class SQSSource(object):
"""Implementation of ISource that receives messages from a SQS queue.
"""
max_delete_delay = 5
def __init__(self, logger, loop, gate, sqs_client, metric_prefix='source'):
self.gate = gate
self.collector = sqs_client
self.logger = logger
self.loop = loop
self.metric_prefix = metric_prefix
self.end_of_input = Event()
self.input_error = Event()
self.state = RUNNING
self._delete_queue = Queue()
self._should_flush_queue = Event()
self.sender_tag = 'sender:%s.%s' % (self.__class__.__module__,
self.__class__.__name__)
self.loop.spawn_callback(self.onInput)
self.loop.spawn_callback(self._onDelete)
@gen.coroutine
def close(self, timeout=None):
self.state = CLOSING
self.logger.warning('Closing source')
yield self._delete_queue.join(timeout)
@gen.coroutine
def _flush_delete_batch(self, batch_size):
delete_batch = [
self._delete_queue.get_nowait()
for pos in range(min(batch_size, self.collector.max_messages))
]
try:
response = yield self.collector.delete_message_batch(*delete_batch)
except SQSError as err:
lmsg = 'Error encountered deleting processed messages in SQS: %s'
self.logger.exception(lmsg, err)
self.input_error.set()
for msg in delete_batch:
self._delete_queue.put_nowait(msg)
else:
if response.Failed:
self.input_error.set()
for req in response.Failed:
self.logger.error('Message failed to delete: %s', req.Id)
self._delete_queue.put_nowait(req)
@gen.coroutine
def _onDelete(self):
respawn = True
while respawn:
try:
qsize = self._delete_queue.qsize()
# This will keep flushing until clear,
# including items that show up in between flushes
while qsize > 0:
yield self._flush_delete_batch(qsize)
qsize = self._delete_queue.qsize()
self._should_flush_queue.clear()
yield self._should_flush_queue.wait()
except Exception as err:
self.logger.exception(err)
self.input_error.set()
respawn = False
@gen.coroutine
def onInput(self):
respawn = True
retry_timeout = INITIAL_TIMEOUT
# We use an algorithm similar to TCP window scaling,
# so that we request fewer messages when we encounter
# back pressure from our gate/drain and request more
# when we flushed a complete batch
window_size = self.collector.max_messages
while respawn:
try:
response = yield self.collector.receive_message_batch(
max_messages=window_size, )
if response.Messages:
# We need to have low latency to delete messages
# we've processed
retry_timeout = INITIAL_TIMEOUT
else:
retry_timeout = min(retry_timeout * 2, MAX_TIMEOUT)
yield gen.sleep(retry_timeout.total_seconds())
sent_full_batch = True
for position, msg in enumerate(response.Messages):
try:
self.gate.put_nowait(msg)
except QueueFull:
self.logger.debug('Gate queue full; yielding')
sent_full_batch = False
# TODO: is it worth trying to batch and schedule
# a flush at this point instead of many
# single deletes?
yield self.gate.put(msg)
self._should_flush_queue.set()
self._delete_queue.put_nowait(msg)
statsd.increment('%s.queued' % self.metric_prefix,
tags=[self.sender_tag])
# If we were able to flush the entire batch without waiting,
# increase our window size to max_messages
if sent_full_batch and \
window_size < self.collector.max_messages:
window_size += 1
# Otherwise ask for less next time
elif not sent_full_batch and window_size > 1:
window_size -= 1
except Exception as err:
self.logger.exception(err)
self.input_error.set()
respawn = False
class FileSystemWatcher(object):
def __init__(self,
watch_paths,
on_changed=None,
interval=1.0,
recursive=True):
"""Constructor.
Args:
watch_paths: A list of filesystem paths to watch for changes.
on_changed: Callback to call when one or more changes to the watch path are detected.
interval: The minimum interval at which to notify about changes (in seconds).
recursive: Should the watch path be monitored recursively for changes?
"""
if isinstance(watch_paths, basestring):
watch_paths = [watch_paths]
watch_paths = [os.path.abspath(path) for path in watch_paths]
for path in watch_paths:
if not os.path.exists(path) or not os.path.isdir(path):
raise MissingFolderError(path)
self.watch_paths = watch_paths
self.interval = interval * 1000.0
self.recursive = recursive
self.periodic_callback = PeriodicCallback(self.check_fs_events,
self.interval)
self.on_changed = on_changed
self.observer = Observer()
for path in self.watch_paths:
self.observer.schedule(WatcherEventHandler(self), path,
self.recursive)
self.started = False
self.fs_event_queue = Queue()
def track_event(self, event):
self.fs_event_queue.put(event)
@gen.coroutine
def check_fs_events(self):
drained_events = []
while self.fs_event_queue.qsize() > 0:
drained_events.append(self.fs_event_queue.get_nowait())
if len(drained_events) > 0 and callable(self.on_changed):
logger.debug(
"Detected %d file system change(s) - triggering callback" %
len(drained_events))
self.on_changed(drained_events)
def start(self):
if not self.started:
self.observer.start()
self.periodic_callback.start()
self.started = True
logger.debug("Started file system watcher for paths:\n%s" %
"\n".join(self.watch_paths))
def shutdown(self, timeout=None):
if self.started:
self.periodic_callback.stop()
self.observer.stop()
self.observer.join(timeout=timeout)
self.started = False
logger.debug("Shut down file system watcher for path:\n%s" %
"\n".join(self.watch_paths))
class AsyncConnection:
INIT_STATUS = "init"
CONNECTING_STATUS = "connecting"
OPEN_STATUS = "open"
CLOSE_STATUS = "close"
TIMEOUT_STATUS = 'timeout'
reconnect_connection = False
logger: Logger = logger
def __init__(self, rabbitmq_url, io_loop, timeout=10):
self._parameters = ConnectionParameters("127.0.0.1") if rabbitmq_url in ["localhost", "127.0.0.1"] else \
URLParameters(rabbitmq_url)
self._io_loop = io_loop
self._timeout = timeout
self._connection_queue = Queue(maxsize=1)
self.current_status = self.INIT_STATUS
@property
def status_ok(self):
return self.current_status != self.CLOSE_STATUS \
and self.current_status != self.TIMEOUT_STATUS
@gen.coroutine
def get_connection(self):
if self.current_status == self.INIT_STATUS:
self.current_status = self.CONNECTING_STATUS
self.connect()
conn = yield self._top()
return conn
def connect(self):
try:
self.try_connect()
except Exception:
self.logger.exception(f"Failed to connect to RabbitMQ.")
@gen.coroutine
def _top(self):
conn = yield self._connection_queue.get()
self._connection_queue.put(conn)
return conn
def _on_timeout(self):
if self.current_status == self.CONNECTING_STATUS:
self.logger.error("Creating connection timed out")
self.current_status = self.TIMEOUT_STATUS
self.stop()
def _open_callback(self, connection):
self.logger.info("Created connection")
self.current_status = self.OPEN_STATUS
self._connection_queue.put(connection)
def _open_error_callback(self, connection, exception):
self.logger.error(f"Open connection with error: {exception}")
self.current_status = self.CLOSE_STATUS
self.reconnect()
def _close_callback(self, connection, reason):
self.logger.error(
f"Closing connection: reason: {reason}. System will exist")
self.current_status = self.CLOSE_STATUS
self.reconnect()
def reconnect(self):
self.reconnect_connection = True
try:
self._connection_queue.get_nowait()
except QueueEmpty:
pass
self.stop()
def stop(self):
if self.reconnect_connection:
self.logger.info("Restarting")
self.current_status = self.INIT_STATUS
self.try_connect()
else:
self.logger.info('Stopping')
self._io_loop.stop()
# untuk buka koneksi dari rabbit mq
def try_connect(self):
self.logger.info("Creating connection to RabbitMQ")
self._io_loop.call_later(self._timeout, self._on_timeout)
if isinstance(self._io_loop, IOLoop):
TornadoConnection(self._parameters,
on_open_callback=self._open_callback,
on_open_error_callback=self._open_error_callback,
on_close_callback=self._close_callback,
custom_ioloop=self._io_loop)
else:
AsyncioConnection(self._parameters,
on_open_callback=self._open_callback,
on_open_error_callback=self._open_error_callback,
on_close_callback=self._close_callback,
custom_ioloop=self._io_loop)
class Application(object):
def __init__(self, routes, node, pipe):
"""
Application instantiates and registers handlers for each message type,
and routes messages to the pre-instantiated instances of each message handler
:param routes: list of tuples in the form of (<message type str>, <MessageHandler class>)
:param node: Node instance of the local node
:param pipe: Instance of multiprocessing.Pipe for communicating with the parent process
"""
# We don't really have to worry about synchronization
# so long as we're careful about explicit context switching
self.nodes = {node.node_id: node}
self.local_node = node
self.handlers = {}
self.tcpclient = TCPClient()
self.gossip_inbox = Queue()
self.gossip_outbox = Queue()
self.sequence_number = 0
if routes:
self.add_handlers(routes)
self.pipe = pipe
self.ioloop = IOLoop.current()
self.add_node_event = Event()
def next_sequence_number(self):
self.sequence_number += 1
return self.sequence_number
@coroutine
def ping_random_node(self):
node = yield self.get_random_node()
LOGGER.debug('{} pinging random node: {}'.format(
self.local_node.node_id, node.node_id))
try:
yield self.ping(node)
except TimeoutError:
self.mark_suspect(node)
@coroutine
def add_node(self, node):
if node.node_id not in self.nodes:
LOGGER.debug('Adding node {} to {}'.format(node, self.nodes))
self.add_node_event.set()
self.nodes[node.node_id] = node
LOGGER.debug('Added node {} to {}'.format(node, self.nodes))
@coroutine
def remove_node(self, node):
if node.node_id in self.nodes:
del self.nodes[node.node_id]
other_nodes = yield self.get_other_nodes
if not other_nodes:
self.add_node_event.clear()
def add_handlers(self, handlers):
for message_type, handler_cls in handlers:
assert message_type in MESSAGE_TYPES, (
'Message type {!r} not found in MESSAGE TYPES {}'.format(
message_type, MESSAGE_TYPES.keys()))
self.handlers[message_type] = handler_cls(self)
def route_stream_message(self, stream, message_type, message):
LOGGER.debug('{!r} received {} message from {!r}'.format(
self, message_type, stream))
message_cls = MESSAGE_TYPES[message_type]
message_obj = message_cls(**message)
handler = self.handlers[message_type]
LOGGER.debug('Routing {} to {}'.format(message_type, handler))
handler(stream, message_obj)
@coroutine
def send_message(self, stream, message):
LOGGER.debug('Sending message {!r} to {}'.format(
message.MESSAGE_TYPE, stream))
try:
yield stream.write(message.to_msgpack)
except StreamClosedError:
LOGGER.warn('Unable to send {} to {} - stream closed'.format(
message.MESSAGE_TYPE, stream))
@coroutine
def _get_next_message(self, stream):
# get the next message from the stream
unpacker = msgpack.Unpacker()
try:
wire_bytes = yield with_timeout(
datetime.timedelta(seconds=PING_TIMEOUT),
stream.read_bytes(4096, partial=True))
except StreamClosedError:
LOGGER.warn(
'Unable to get next message from {} - stream closed'.format(
stream))
else:
unpacker.feed(wire_bytes)
LOGGER.debug('Deserializing object from stream {}'.format(stream))
message = unpacker.next()
message.pop('type')
raise Return(message)
@coroutine
def ping(self, node):
"""
Ping a node
:param node: Instance of Node to ping
:returns: Boolean, True if successful/False if fail
"""
host = node.addr
port = node.port
LOGGER.debug('pinging {}:{}'.format(host, port))
ping = Ping(seqno=self.next_sequence_number(),
node=node,
sender=self.local_node)
# Connect to the node
try:
stream = yield self.tcpclient.connect(host, port)
except StreamClosedError:
LOGGER.error(
'Unable to connect from {} to {} (pinging host)'.format(
self.local_node.node_id, node.node_id))
raise Return(False)
try:
# Send the ping
LOGGER.debug('Sending {!r} to {!r}'.format(ping.MESSAGE_TYPE,
node))
yield self.send_message(stream, ping)
# Wait for an ACK message in response
LOGGER.debug('Getting next message from {}:{}'.format(host, port))
message = yield self._get_next_message(stream)
if message is None:
raise Return(False)
ack = Ack(**message)
LOGGER.debug('Received {!r} from {!r} (response to {!r})'.format(
ack.MESSAGE_TYPE, node.node_id, ping.MESSAGE_TYPE))
# Check that the ACK sequence number matches the PING sequence number
if ack.seqno == ping.seqno:
LOGGER.debug(
'Sequence number matches. Node {} looks good to !'.format(
node.node_id, self.local_node.node_id))
# Process the gossip messages tacked onto the ACK message's payload
for message in ack.payload:
try:
self.gossip_inbox.put_nowait(message)
except QueueFull:
LOGGER.error(
'Unable to add {} message from {} to gossip inbox'.
format(message.MESSAGE_TYPE, node.node_id))
# mark the node as ALIVE in self.nodes
self.mark_alive(node)
# Send gossip that this node is alive
self.queue_gossip_send(Alive(node=node,
sender=self.local_node))
raise Return(True)
else:
raise Return(False)
finally:
stream.close()
@coroutine
def ack(self, stream, seqno):
payload = []
for _ in xrange(ACK_PAYLOAD_SIZE):
try:
gossip = self.gossip_outbox.get_nowait()
payload.append(gossip)
except QueueEmpty:
break
ack = Ack(seqno=seqno, payload=payload)
LOGGER.debug('Trying to send ack: {}'.format(ack))
try:
yield stream.write(ack.to_msgpack)
except StreamClosedError:
LOGGER.error(
'Unable to connect from {} to stream (acking PING)'.format(
self.local_node.node_id))
LOGGER.debug('Sent ack to {}'.format(stream))
@coroutine
def _change_node_state(self, node, state):
"""
Because Tornado has explicit context switching, we don't need to worry much about synchronization here
"""
LOGGER.debug('{} knows about {}: {}'.format(self.local_node.node_id,
node.node_id, state))
self.add_node(node)
self.nodes[node.node_id].state = state
@coroutine
def mark_alive(self, node):
if node.node_id != self.local_node.node_id:
LOGGER.debug('Marking {} ALIVE'.format(node.node_id))
self._change_node_state(node, State.ALIVE)
@coroutine
def mark_dead(self, node):
self._change_node_state(node, State.DEAD)
@coroutine
def mark_suspect(self, node):
self._change_node_state(node, State.SUSPECT)
@coroutine
def ingest_gossip_inbox(self):
while True:
LOGGER.debug('checking inbox')
message = yield self.gossip_inbox.get()
LOGGER.debug('Received message {} from gossip inbox'.format(
message.MESSAGE_TYPE))
if message.MESSAGE_TYPE == Alive.MESSAGE_TYPE:
self.mark_alive(message.sender)
self.mark_alive(message.node)
self.queue_gossip_send(message)
elif message.MESSAGE_TYPE == Suspect.MESSAGE_TYPE:
self.mark_alive(message.sender)
self.mark_suspect(message.node)
self.queue_gossip_send(message)
elif message.MESSAGE_TYPE == Dead.MESSAGE_TYPE:
self.mark_alive(message.sender)
self.mark_dead(message.node)
self.queue_gossip_send(message)
@coroutine
def queue_gossip_send(self, message):
"""
If the message is gossipable, add it to the outbox
"""
try:
next_incarnation = message.next_incarnation
next_incarnation.sender = self.local_node
except message.MaxIncarnationsReached:
LOGGER.debug(
'Max incarnations reached for {}! No gossip 4 u'.format(
message.MESSAGE_TYPE))
else:
LOGGER.debug('Enqueuing {} gossips for {}'.format(
GOSSIP_PEERS, message))
for _ in xrange(GOSSIP_PEERS):
yield self.gossip_outbox.put(next_incarnation)
@coroutine
def send_buffered_gossip(self):
while True:
random_node = yield self.get_random_node()
message = yield self.gossip_outbox.get()
LOGGER.debug('{} connecting to {} for gossip'.format(
self.local_node, random_node))
try:
stream = yield self.tcpclient.connect(random_node.addr,
random_node.port)
except StreamClosedError:
LOGGER.error(
'Unable to connect from {} to {} (sending gossip)'.format(
self.local_node.node_id, random_node.node_id))
LOGGER.warning('Putting the gossip back on our queue')
try:
self.gossip_outbox.put_nowait(message)
except QueueFull:
LOGGER.error(
'Unable to put gossip back onto the queue. Giving up!')
else:
try:
LOGGER.debug('{} gossipping with {}'.format(
self.local_node.node_id, random_node.node_id))
yield self.send_message(stream, message)
finally:
stream.close()
@coroutine
def get_other_nodes(self, exclude=None):
if exclude is None:
exclude = (self.local_node, )
exclude_node_ids = [n.node_id for n in exclude]
raise Return([n for n in self.nodes if n not in exclude_node_ids])
@coroutine
def get_random_node(self, exclude=None):
LOGGER.debug('Waiting for more nodes')
yield self.add_node_event.wait()
LOGGER.debug('Getting non-self random node')
other_nodes = yield self.get_other_nodes(exclude=exclude)
LOGGER.debug('{} got something! choices: {}'.format(
self.local_node.node_id, other_nodes))
assert other_nodes
node_id = random.choice(other_nodes)
raise Return(self.nodes[node_id])
class Application(object):
def __init__(self, routes, node, pipe):
"""
Application instantiates and registers handlers for each message type,
and routes messages to the pre-instantiated instances of each message handler
:param routes: list of tuples in the form of (<message type str>, <MessageHandler class>)
:param node: Node instance of the local node
:param pipe: Instance of multiprocessing.Pipe for communicating with the parent process
"""
# We don't really have to worry about synchronization
# so long as we're careful about explicit context switching
self.nodes = {node.node_id: node}
self.local_node = node
self.handlers = {}
self.tcpclient = TCPClient()
self.gossip_inbox = Queue()
self.gossip_outbox = Queue()
self.sequence_number = 0
if routes:
self.add_handlers(routes)
self.pipe = pipe
self.ioloop = IOLoop.current()
self.add_node_event = Event()
def next_sequence_number(self):
self.sequence_number += 1
return self.sequence_number
@coroutine
def ping_random_node(self):
node = yield self.get_random_node()
LOGGER.debug('{} pinging random node: {}'.format(self.local_node.node_id,
node.node_id))
try:
yield self.ping(node)
except TimeoutError:
self.mark_suspect(node)
@coroutine
def add_node(self, node):
if node.node_id not in self.nodes:
LOGGER.debug('Adding node {} to {}'.format(node, self.nodes))
self.add_node_event.set()
self.nodes[node.node_id] = node
LOGGER.debug('Added node {} to {}'.format(node, self.nodes))
@coroutine
def remove_node(self, node):
if node.node_id in self.nodes:
del self.nodes[node.node_id]
other_nodes = yield self.get_other_nodes
if not other_nodes:
self.add_node_event.clear()
def add_handlers(self, handlers):
for message_type, handler_cls in handlers:
assert message_type in MESSAGE_TYPES, (
'Message type {!r} not found in MESSAGE TYPES {}'.format(
message_type,
MESSAGE_TYPES.keys()
)
)
self.handlers[message_type] = handler_cls(self)
def route_stream_message(self, stream, message_type, message):
LOGGER.debug('{!r} received {} message from {!r}'.format(self, message_type, stream))
message_cls = MESSAGE_TYPES[message_type]
message_obj = message_cls(**message)
handler = self.handlers[message_type]
LOGGER.debug('Routing {} to {}'.format(message_type, handler))
handler(stream, message_obj)
@coroutine
def send_message(self, stream, message):
LOGGER.debug('Sending message {!r} to {}'.format(message.MESSAGE_TYPE, stream))
try:
yield stream.write(message.to_msgpack)
except StreamClosedError:
LOGGER.warn('Unable to send {} to {} - stream closed'.format(message.MESSAGE_TYPE, stream))
@coroutine
def _get_next_message(self, stream):
# get the next message from the stream
unpacker = msgpack.Unpacker()
try:
wire_bytes = yield with_timeout(
datetime.timedelta(seconds=PING_TIMEOUT),
stream.read_bytes(4096, partial=True)
)
except StreamClosedError:
LOGGER.warn('Unable to get next message from {} - stream closed'.format(stream))
else:
unpacker.feed(wire_bytes)
LOGGER.debug('Deserializing object from stream {}'.format(stream))
message = unpacker.next()
message.pop('type')
raise Return(message)
@coroutine
def ping(self, node):
"""
Ping a node
:param node: Instance of Node to ping
:returns: Boolean, True if successful/False if fail
"""
host = node.addr
port = node.port
LOGGER.debug('pinging {}:{}'.format(host, port))
ping = Ping(seqno=self.next_sequence_number(),
node=node,
sender=self.local_node)
# Connect to the node
try:
stream = yield self.tcpclient.connect(host, port)
except StreamClosedError:
LOGGER.error('Unable to connect from {} to {} (pinging host)'.format(self.local_node.node_id, node.node_id))
raise Return(False)
try:
# Send the ping
LOGGER.debug('Sending {!r} to {!r}'.format(ping.MESSAGE_TYPE, node))
yield self.send_message(stream, ping)
# Wait for an ACK message in response
LOGGER.debug('Getting next message from {}:{}'.format(host, port))
message = yield self._get_next_message(stream)
if message is None:
raise Return(False)
ack = Ack(**message)
LOGGER.debug('Received {!r} from {!r} (response to {!r})'.format(ack.MESSAGE_TYPE,
node.node_id,
ping.MESSAGE_TYPE))
# Check that the ACK sequence number matches the PING sequence number
if ack.seqno == ping.seqno:
LOGGER.debug('Sequence number matches. Node {} looks good to !'.format(node.node_id,
self.local_node.node_id))
# Process the gossip messages tacked onto the ACK message's payload
for message in ack.payload:
try:
self.gossip_inbox.put_nowait(message)
except QueueFull:
LOGGER.error('Unable to add {} message from {} to gossip inbox'.format(message.MESSAGE_TYPE,
node.node_id))
# mark the node as ALIVE in self.nodes
self.mark_alive(node)
# Send gossip that this node is alive
self.queue_gossip_send(
Alive(node=node, sender=self.local_node)
)
raise Return(True)
else:
raise Return(False)
finally:
stream.close()
@coroutine
def ack(self, stream, seqno):
payload = []
for _ in xrange(ACK_PAYLOAD_SIZE):
try:
gossip = self.gossip_outbox.get_nowait()
payload.append(gossip)
except QueueEmpty:
break
ack = Ack(seqno=seqno, payload=payload)
LOGGER.debug('Trying to send ack: {}'.format(ack))
try:
yield stream.write(ack.to_msgpack)
except StreamClosedError:
LOGGER.error('Unable to connect from {} to stream (acking PING)'.format(self.local_node.node_id))
LOGGER.debug('Sent ack to {}'.format(stream))
@coroutine
def _change_node_state(self, node, state):
"""
Because Tornado has explicit context switching, we don't need to worry much about synchronization here
"""
LOGGER.debug('{} knows about {}: {}'.format(self.local_node.node_id, node.node_id, state))
self.add_node(node)
self.nodes[node.node_id].state = state
@coroutine
def mark_alive(self, node):
if node.node_id != self.local_node.node_id:
LOGGER.debug('Marking {} ALIVE'.format(node.node_id))
self._change_node_state(node, State.ALIVE)
@coroutine
def mark_dead(self, node):
self._change_node_state(node, State.DEAD)
@coroutine
def mark_suspect(self, node):
self._change_node_state(node, State.SUSPECT)
@coroutine
def ingest_gossip_inbox(self):
while True:
LOGGER.debug('checking inbox')
message = yield self.gossip_inbox.get()
LOGGER.debug('Received message {} from gossip inbox'.format(message.MESSAGE_TYPE))
if message.MESSAGE_TYPE == Alive.MESSAGE_TYPE:
self.mark_alive(message.sender)
self.mark_alive(message.node)
self.queue_gossip_send(message)
elif message.MESSAGE_TYPE == Suspect.MESSAGE_TYPE:
self.mark_alive(message.sender)
self.mark_suspect(message.node)
self.queue_gossip_send(message)
elif message.MESSAGE_TYPE == Dead.MESSAGE_TYPE:
self.mark_alive(message.sender)
self.mark_dead(message.node)
self.queue_gossip_send(message)
@coroutine
def queue_gossip_send(self, message):
"""
If the message is gossipable, add it to the outbox
"""
try:
next_incarnation = message.next_incarnation
next_incarnation.sender = self.local_node
except message.MaxIncarnationsReached:
LOGGER.debug('Max incarnations reached for {}! No gossip 4 u'.format(message.MESSAGE_TYPE))
else:
LOGGER.debug('Enqueuing {} gossips for {}'.format(GOSSIP_PEERS, message))
for _ in xrange(GOSSIP_PEERS):
yield self.gossip_outbox.put(next_incarnation)
@coroutine
def send_buffered_gossip(self):
while True:
random_node = yield self.get_random_node()
message = yield self.gossip_outbox.get()
LOGGER.debug('{} connecting to {} for gossip'.format(self.local_node, random_node))
try:
stream = yield self.tcpclient.connect(random_node.addr, random_node.port)
except StreamClosedError:
LOGGER.error('Unable to connect from {} to {} (sending gossip)'.format(self.local_node.node_id,
random_node.node_id))
LOGGER.warning('Putting the gossip back on our queue')
try:
self.gossip_outbox.put_nowait(message)
except QueueFull:
LOGGER.error('Unable to put gossip back onto the queue. Giving up!')
else:
try:
LOGGER.debug('{} gossipping with {}'.format(self.local_node.node_id, random_node.node_id))
yield self.send_message(stream, message)
finally:
stream.close()
@coroutine
def get_other_nodes(self, exclude=None):
if exclude is None:
exclude = (self.local_node,)
exclude_node_ids = [n.node_id for n in exclude]
raise Return([n for n in self.nodes if n not in exclude_node_ids])
@coroutine
def get_random_node(self, exclude=None):
LOGGER.debug('Waiting for more nodes')
yield self.add_node_event.wait()
LOGGER.debug('Getting non-self random node')
other_nodes = yield self.get_other_nodes(exclude=exclude)
LOGGER.debug('{} got something! choices: {}'.format(self.local_node.node_id, other_nodes))
assert other_nodes
node_id = random.choice(other_nodes)
raise Return(self.nodes[node_id])
