class QueueHandler(logging.Handler):
def __init__(self):
logging.Handler.__init__(self)
self.queue = Queue()
self._records_lost = 0
def _put_record(self, record):
self.format(record)
record.msg = record.message
record.args = None
record.exc_info = None
self.queue.put_nowait(record)
def _try_to_report_lost_records(self):
if self._records_lost:
try:
record = _LOGGER.makeRecord(
_LOGGER.name, logging.WARNING, __file__, 0,
'QueueHandler has lost %s log records',
(self._records_lost, ), None, 'emit')
self._put_record(record)
self._records_lost = 0
except Exception:
pass
def emit(self, record):
try:
self._put_record(record)
self._try_to_report_lost_records()
except Exception:
self._records_lost += 1
@property
def records_lost(self):
return self._records_lost
class ThreadPool(object):
def __init__(self, workers=10):
self.queue = Queue()
self.workers = []
self.tasks = []
for worker in range(workers):
self.workers.append(Worker(self.queue))
def add(self, ident, func, args=None, kwargs=None):
if args is None:
args = ()
if kwargs is None:
kwargs = {}
task = (ident, func, args, kwargs)
self.tasks.append(ident)
self.queue.put_nowait(task)
def join(self):
for worker in self.workers:
worker.start()
results = defaultdict(list)
for worker in self.workers:
worker.join()
for k, v in six.iteritems(worker.results):
results[k].extend(v)
return results
class NLSocketPool(object):
"""Pool of netlink sockets."""
def __init__(self, size):
if size <= 0:
raise ValueError('Invalid socket pool size %r. Must be positive')
self._semaphore = BoundedSemaphore(size)
self._sockets = Queue(maxsize=size)
@contextmanager
def socket(self):
"""Returns a socket from the pool (creating it when needed)."""
with self._semaphore:
try:
sock = self._sockets.get_nowait()
except Empty:
sock = _open_socket()
try:
yield sock
finally:
self._sockets.put_nowait(sock)
class NLSocketPool(object):
"""Pool of netlink sockets."""
def __init__(self, size):
if size <= 0:
raise ValueError('Invalid socket pool size %r. Must be positive')
self._semaphore = BoundedSemaphore(size)
self._sockets = Queue(maxsize=size)
@contextmanager
def socket(self):
"""Returns a socket from the pool (creating it when needed)."""
with self._semaphore:
try:
sock = self._sockets.get_nowait()
except Empty:
sock = _open_socket()
try:
yield sock
finally:
self._sockets.put_nowait(sock)
class AmqpSubscriber(Subscriber):
def __init__(self, amqp_chan, exchanges):
self.channel = amqp_chan
self.messages = Queue(maxsize=0)
qname, _, _ = self.channel.queue_declare()
for exchange in exchanges:
self.channel.queue_bind(qname, exchange)
self.channel.basic_consume(queue=qname, callback=self.callback)
def callback(self, msg):
self.channel.basic_ack(msg.delivery_tag)
self.messages.put_nowait(msg.body)
def __iter__(self):
return self
def next(self):
while self.messages.empty():
self.channel.wait()
return self.messages.get_nowait()
__next__ = next # PY3
class AmqpSubscriber(Subscriber):
def __init__(self, amqp_chan, exchanges):
self.channel = amqp_chan
self.messages = Queue(maxsize=0)
qname, _, _ = self.channel.queue_declare()
for exchange in exchanges:
self.channel.queue_bind(qname, exchange)
self.channel.basic_consume(queue=qname, callback=self.callback)
def callback(self, msg):
self.channel.basic_ack(msg.delivery_tag)
self.messages.put_nowait(msg.body)
def __iter__(self):
return self
def next(self):
while self.messages.empty():
self.channel.wait()
return self.messages.get_nowait()
__next__ = next # PY3
class KeyboardCapture(threading.Thread):
"""Implementation of KeyboardCapture for OSX."""
_KEYBOARD_EVENTS = set([kCGEventKeyDown, kCGEventKeyUp])
def __init__(self):
threading.Thread.__init__(self, name="KeyboardEventTapThread")
self._loop = None
self._event_queue = Queue() # Drained by event handler thread.
self._suppressed_keys = set()
self.key_down = lambda key: None
self.key_up = lambda key: None
# Returning the event means that it is passed on
# for further processing by others.
#
# Returning None means that the event is intercepted.
#
# Delaying too long in returning appears to cause the
# system to ignore the tap forever after
# (https://github.com/openstenoproject/plover/issues/484#issuecomment-214743466).
#
# This motivates pushing callbacks to the other side
# of a queue of received events, so that we can return
# from this callback as soon as possible.
def callback(proxy, event_type, event, reference):
SUPPRESS_EVENT = None
PASS_EVENT_THROUGH = event
# Don't pass on meta events meant for this event tap.
is_unexpected_event = event_type not in self._KEYBOARD_EVENTS
if is_unexpected_event:
if event_type == kCGEventTapDisabledByTimeout:
# Re-enable the tap and hope we act faster next time
CGEventTapEnable(self._tap, True)
plover.log.warning(
"Keystrokes may have been missed. " +
"Keyboard event tap has been re-enabled. ")
return SUPPRESS_EVENT
# Don't intercept the event if it has modifiers, allow
# Fn and Numeric flags so we can suppress the arrow and
# extended (home, end, etc...) keys.
suppressible_modifiers = (kCGEventFlagMaskNumericPad
| kCGEventFlagMaskSecondaryFn
| kCGEventFlagMaskNonCoalesced)
has_nonsupressible_modifiers = \
CGEventGetFlags(event) & ~suppressible_modifiers
if has_nonsupressible_modifiers and event_type == kCGEventKeyDown:
return PASS_EVENT_THROUGH
keycode = CGEventGetIntegerValueField(event,
kCGKeyboardEventKeycode)
key = KEYCODE_TO_KEY.get(keycode)
self._async_dispatch(key, event_type)
if key in self._suppressed_keys:
return SUPPRESS_EVENT
return PASS_EVENT_THROUGH
self._tap = CGEventTapCreate(
kCGSessionEventTap, kCGHeadInsertEventTap,
kCGEventTapOptionDefault,
CGEventMaskBit(kCGEventKeyDown) | CGEventMaskBit(kCGEventKeyUp),
callback, None)
if self._tap is None:
# Todo(hesky): See if there is a nice way to show the user what's
# needed (or do it for them).
raise Exception("Enable access for assistive devices.")
CGEventTapEnable(self._tap, False)
def run(self):
source = CFMachPortCreateRunLoopSource(None, self._tap, 0)
handler_thread = threading.Thread(target=self._event_handler,
name="KeyEventDispatcher")
handler_thread.start()
self._loop = CFRunLoopGetCurrent()
CFRunLoopAddSource(self._loop, source, kCFRunLoopCommonModes)
CGEventTapEnable(self._tap, True)
CFRunLoopRun()
# Wake up event handler.
self._event_queue.put_nowait(None)
handler_thread.join()
CFMachPortInvalidate(self._tap)
CFRelease(self._tap)
CFRunLoopSourceInvalidate(source)
def cancel(self):
CFRunLoopStop(self._loop)
self.join()
self._loop = None
def suppress_keyboard(self, suppressed_keys=()):
self._suppressed_keys = set(suppressed_keys)
def _async_dispatch(self, key, event_type):
"""
Dispatches a key string in KEYCODE_TO_KEY.values() and a CGEventType
to the appropriate KeyboardCapture callback
without blocking execution of its caller.
"""
if key is None:
return
is_keyup = event_type == kCGEventKeyUp
pair = (key, is_keyup)
self._event_queue.put_nowait(pair)
def _event_handler(self):
"""
Event dispatching thread launched during run().
Loops until None is received from _event_queue.
Avoids busy-waiting by blocking on _event_queue.
In normal operation, it gets a pair of
(key_string, is_keyup_bool) from _event_queue
and routes the string to self.key_up or self.key_down,
then waits for a new pair to arrive.
"""
while True:
pair = self._event_queue.get(block=True, timeout=None)
if pair is None:
return
key, is_keyup = pair
handler = self.key_up if is_keyup else self.key_down
handler(key)
class JsonRpcServer(object):
log = logging.getLogger("jsonrpc.JsonRpcServer")
FILTERED_METHODS = frozenset(['Host.getAllVmStats'])
"""
Creates new JsonrRpcServer by providing a bridge, timeout in seconds
which defining how often we should log connections stats and thread
factory.
"""
def __init__(self, bridge, timeout, cif, threadFactory=None):
self._bridge = bridge
self._cif = cif
self._workQueue = Queue()
self._threadFactory = threadFactory
self._timeout = timeout
self._next_report = monotonic_time() + self._timeout
self._counter = 0
def queueRequest(self, req):
self._workQueue.put_nowait(req)
"""
Aggregates number of requests received by vdsm. Each request from
a batch is added separately. After time defined by timeout we log
number of requests.
"""
def _attempt_log_stats(self):
self._counter += 1
if monotonic_time() > self._next_report:
self.log.info('%s requests processed during %s seconds',
self._counter, self._timeout)
self._next_report += self._timeout
self._counter = 0
def _serveRequest(self, ctx, req):
self._attempt_log_stats()
logLevel = logging.DEBUG
suppress_logging = req.method in self.FILTERED_METHODS
# VDSM should never respond to any request before all information about
# running VMs is recovered, see https://bugzilla.redhat.com/1339291
if not self._cif.ready:
self.log.info("In recovery, ignoring '%s' in bridge with %s",
req.method, req.params)
# TODO: take the response from the exception instead of via errCode
ctx.requestDone(JsonRpcResponse(errCode['recovery'], None, req.id))
return
self.log.log(logLevel, "Calling '%s' in bridge with %s",
req.method, req.params)
try:
method = self._bridge.dispatch(req.method)
except JsonRpcMethodNotFoundError as e:
if req.isNotification():
return
ctx.requestDone(JsonRpcResponse(None, e, req.id))
return
try:
params = req.params
self._bridge.register_server_address(ctx.server_address)
if isinstance(req.params, list):
res = method(*params)
else:
res = method(**params)
self._bridge.unregister_server_address()
except JsonRpcError as e:
ctx.requestDone(JsonRpcResponse(None, e, req.id))
except Exception as e:
self.log.exception("Internal server error")
ctx.requestDone(JsonRpcResponse(None,
JsonRpcInternalError(str(e)),
req.id))
else:
res = True if res is None else res
log_res = "(suppressed)" if suppress_logging else res
self.log.log(logLevel, "Return '%s' in bridge with %s",
req.method, log_res)
ctx.requestDone(JsonRpcResponse(res, None, req.id))
@traceback(on=log.name)
def serve_requests(self):
while True:
obj = self._workQueue.get()
if obj is None:
break
client, server_address, msg = obj
self._parseMessage(client, server_address, msg)
def _parseMessage(self, client, server_address, msg):
ctx = _JsonRpcServeRequestContext(client, server_address)
try:
rawRequests = json.loads(msg)
except:
ctx.addResponse(JsonRpcResponse(None, JsonRpcParseError(), None))
ctx.sendReply()
return
if isinstance(rawRequests, list):
# Empty batch request
if len(rawRequests) == 0:
ctx.addResponse(
JsonRpcResponse(None,
JsonRpcInvalidRequestError(
'request batch is empty',
rawRequests),
None))
ctx.sendReply()
return
else:
# From this point on we know it's always a list
rawRequests = [rawRequests]
# JSON Parsed handling each request
requests = []
for rawRequest in rawRequests:
try:
req = JsonRpcRequest.fromRawObject(rawRequest)
requests.append(req)
except JsonRpcError as err:
ctx.addResponse(JsonRpcResponse(None, err, None))
except:
ctx.addResponse(JsonRpcResponse(None,
JsonRpcInternalError(),
None))
ctx.setRequests(requests)
# No request was built successfully or is only notifications
if ctx.counter == 0:
ctx.sendReply()
for request in requests:
self._runRequest(ctx, request)
def _runRequest(self, ctx, request):
if self._threadFactory is None:
self._serveRequest(ctx, request)
else:
try:
self._threadFactory(partial(self._serveRequest, ctx, request))
except Exception as e:
self.log.exception("could not allocate request thread")
ctx.requestDone(
JsonRpcResponse(
None,
JsonRpcInternalError(
str(e)
),
request.id
)
)
def stop(self):
self.log.info("Stopping JsonRPC Server")
self._workQueue.put_nowait(None)
class Manager:
"""
Attribute:
open_handle: A open-handle for handler.
max_task: Specify the upper limit of running task at the same time.
open_args: opener default args.
open_kwargs: opener default kwargs.
daemon: Default make client thread to be daemon thread if True.
"""
def __init__(self,
max_task,
open_handle,
open_args=(),
open_kwargs=None,
daemon=False):
self._full_queue = []
self._queue = TaskQueue(unopened=[],
opening=[],
ready=[],
queue=[],
running=[],
paused=[],
finished=[],
failed=[])
self._open_handle = open_handle
self._max_task = max_task
if not open_kwargs:
open_kwargs = {}
self._open_args = open_args
self._open_kwargs = open_kwargs
self._process_mgr = None
self._thread_queue = ThreadQueue()
self._process_queue = None
self._thread_collector = ThreadCollector(daemon)
self._queue_started = False
self._queue_lock = Lock()
self._process_collector = {}
self._closed = False
def putrequest(self, request, enqueue=True):
""" Put a Request into Manager queue.
Return task id corresponding to the request.
"""
if not self._process_queue:
raise ChildProcessDisable('cannot put a request running on '
'child process mode without enabling.')
if request in self._full_queue:
raise ValueError('request is already in the task queue.')
index = len(self._full_queue)
wrapper = RequestWrapper(id=index,
source=request,
callback=ThreadQueue(),
child_process=request.child_process)
self._full_queue.append(wrapper)
self._queue.unopened.append(index)
if enqueue:
self.enqueue(index)
return index
def enable_child_process_mode(self):
if not self._process_mgr:
self._process_mgr = ProcessManager()
self._process_queue = self._process_mgr.Queue()
self._process_collector[-1] = self._process_mgr._process
def remove(self, task_id):
""" Remove task by task id. """
self._border_check(task_id)
self._full_queue[task_id] = None
def puthandler(self, handler):
""" Put a Handler into Manager queue.
Return task id corresponding to the handler.
Warning: Not supported in child_process mode.
"""
if handler in self._full_queue:
raise ValueError('handler is already in the task queue.')
index = len(self._full_queue)
wrapper = HandlerWrapper(id=index, source=handler)
self._full_queue.append(wrapper)
handler.install_external_callback_queue(
SignalQueue(id=index, queue=self._thread_queue))
self._queue.ready.append(index)
return index
def get(self, task_id):
""" Return HandlerWrapper of the task specified by id.
Warning: this method does not supported by running in child process mode.
"""
return self._full_queue[task_id]
def process_info(self):
infos = {}
for k, v in self._process_collector.items():
infos[k] = ProcessInfo(name=v.name, pid=v.pid, ident=v.ident)
return infos
def start_all(self):
for i in self._queue.ready:
self.start(i)
for i in self._queue.paused:
self.start(i)
def get_all(self):
for i in self._full_queue:
if i is not None:
yield i
def enqueue(self, task_id):
""" Make task into the plan queue.
When queue is started, manager would run the tasks
in the queue automatically until queue is empty.
"""
self._border_check(task_id)
with self._queue_lock:
if task_id in self._queue.finished:
raise RuntimeError('cannot queue a finished task.')
self.__remove_queue_task(task_id)
self._queue.queue.append(task_id)
def getinstspeed(self, task_id=None):
if task_id is not None:
if not self._opened_info_checking(task_id):
return 0
return self._full_queue[task_id].source.getinstspeed()
else:
speed = 0
for i in self._queue.running:
speed += self._full_queue[i].source.getinstspeed()
return speed
def getavgspeed(self, task_id=None):
if task_id is not None:
if not self._opened_info_checking(task_id):
return 0
return self._full_queue[task_id].source.getavgspeed()
else:
speed = 0
for i in self._queue.running:
speed += self._full_queue[i].source.getavgspeed()
return speed
def getincbyte(self, task_id=None):
if task_id is not None:
if not self._opened_info_checking(task_id):
return 0
return self._full_queue[task_id].source.get_go_inc()
else:
incbyte = 0
for i in self._queue.running:
incbyte += self._full_queue[i].source.get_go_inc()
return incbyte
def get_remain_time(self, task_id=None):
if task_id is not None:
if not self._opened_info_checking(task_id):
return float('inf')
return self._full_queue[task_id].source.get_time_left()
else:
remain = 0
for i in self._queue.running:
remain += self._full_queue[i].source.get_time_left()
return remain
def get_remain_byte(self, task_id=None):
if task_id is not None:
if not self._opened_info_checking(task_id):
return float('inf')
return self._full_queue[task_id].source.get_byte_left()
else:
remain = 0
for i in self._queue.running:
remain += self._full_queue[i].source.get_byte_left()
return remain
def getfileinfo(self, task_id):
if not self._opened_info_checking(task_id):
return None
return self._full_queue[task_id].source.getfileinfo()
def geturlinfo(self, task_id):
if not self._opened_info_checking(task_id):
return None
return self._full_queue[task_id].source.geturlinfo_all()
def start_queue(self):
""" Start running plan queue.
The max number of running task is limited by self._max_task.
"""
self._queue_started = True
cur_task_len = len(self._queue.running)
cur_task_len += len(self._queue.running)
if cur_task_len < self._max_task:
for i in range(self._max_task - cur_task_len):
if self._queue.queue:
task_id = self._queue.queue[0]
self.start(task_id)
else:
self.stop_queue()
break
def stop_queue(self):
""" Stop running plan queue.
This method does not stop the running/opening task.
"""
self._queue_started = False
def start(self, task_id):
self._border_check(task_id)
if type(self._full_queue[task_id]) is RequestWrapper:
self._full_queue[task_id].after(self.start,
args=(task_id, ),
force=True)
self.__remove_queue_task(task_id)
self._queue.opening.append(task_id)
self.open(task_id, block=False)
else:
self._full_queue[task_id].source.start()
self._send_ctrl_signal(SIGNAL_TASK_START, task_id)
self._check_controller_state()
def resume(self, task_id):
""" Resume the failed task.
Only make failed task out of failed queue, cannot reopen/rerun the task.
"""
if task_id not in self._queue.failed:
raise RuntimeError('task <%d> is no need to resume.' % task_id)
self._queue.failed.remove(task_id)
if type(self._full_queue[task_id]) is RequestWrapper:
self._queue.unopened.append(task_id)
else:
self._queue.ready.append(task_id)
def pause(self, task_id):
self._border_check(task_id)
self._send_ctrl_signal(SIGNAL_TASK_PAUSE, task_id)
stop = pause
def pause_all(self, *, block=True):
for i in self._queue.running:
self.pause(i)
for i in self._thread_collector.get_group(GROUP_OPEN_CTRL):
task_id = i.owner
self.pause(task_id)
if block:
for i in self._queue.running:
self._full_queue[i].source.join()
stop_all = pause_all
def open(self, task_id, *, block=True):
self._border_check(task_id)
self._check_controller_state()
if type(self._full_queue[task_id]) is not RequestWrapper:
raise TypeError('opening task should be a type of request.')
if block:
return self._open(task_id)
else:
self._thread_collector.put(self._open,
GROUP_OPEN,
args=(task_id, ),
owner=task_id).start()
def open_all(self, *, sequential=False, block=True):
for i in self._queue.unopened:
self.open(i, block=sequential)
if block:
self._thread_collector.wait(GROUP_OPEN)
def is_all_finish(self):
if not self.is_idle():
return False
if self._queue.paused or self._queue.failed or self._queue.unopened:
return False
return True
def is_idle(self):
""" Return True if all running and opening task are over. """
if self._thread_collector.get_all():
return False
if self._queue.running:
return False
return True
def get_queue(self):
return tuple(self._queue.queue)
def get_unopened(self):
return tuple(self._queue.unopened)
def get_finished(self):
return tuple(self._queue.finished)
def get_running(self):
return tuple(self._queue.running)
def get_failed(self):
return tuple(self._queue.failed)
def get_paused(self):
return tuple(self._queue.paused)
def is_finish(self, task_id):
self._border_check(task_id)
return self._full_queue[task_id].source.is_finished()
def close(self):
if not self.is_idle():
raise RuntimeError('cannot close a running manager.')
if self._closed:
raise RuntimeError('manager has already closed.')
if self._process_mgr:
self._process_mgr.shutdown()
self._process_mgr = None
del self._process_collector[-1]
for k, v in self._process_collector.items():
if not v._close:
v.terminate()
self._closed = True
def _opening_control_thread(self, task_id, handler):
callback = self._full_queue[task_id].callback
while True:
signal = callback.get()
if signal.id == ID_TASK_OPEN:
self._full_queue[task_id] = signal.content
self._send_ctrl_signal(SIGNAL_TASK_OPEN, task_id)
callback.put_nowait(SIGNAL_CALLBACK_END)
elif signal.id == ID_TASK_PAUSE:
handler.pause(block=False)
self._send_ctrl_signal(SIGNAL_TASK_PAUSE, task_id)
callback.put_nowait(SIGNAL_CALLBACK_END)
elif signal.id == ID_TASK_FAIL:
self._send_ctrl_signal(SIGNAL_TASK_FAIL, task_id,
signal.content)
callback.put_nowait(SIGNAL_CALLBACK_END)
elif signal.id == ID_CALLBACK_END:
while not callback.empty():
callback.get_nowait()
callback.task_done()
break
callback.task_done()
callback.task_done()
def _check_controller_state(self):
if not self._thread_collector.get_group(GROUP_CTRL_T):
self._thread_collector.put(self._manager_control_thread,
GROUP_CTRL_T,
args=(self._thread_queue, ),
owner=self._thread_queue).start()
if not self._thread_collector.get_group(GROUP_CTRL_P):
self._thread_collector.put(self._manager_control_thread,
GROUP_CTRL_P,
args=(self._process_queue, ),
owner=self._process_queue).start()
def _manager_control_thread(self, queue):
while True:
signal = queue.get()
task_id = signal.content.id
content = signal.content.content
if signal.id == ID_TASK_FINISH:
self.__finished_task_handler(task_id)
elif signal.id == ID_TASK_OPEN:
self.__opened_task_handler(task_id)
elif signal.id in (ID_TASK_EXCEPTION, ID_TASK_FAIL):
self.__failed_task_handler(task_id, content)
elif signal.id == ID_TASK_STOP:
self.__stopped_task_handler(task_id)
elif signal.id == ID_TASK_PAUSE:
self.__paused_task_handler(signal.content)
elif signal.id == ID_TASK_START:
self.__started_task_handler(task_id)
elif signal.id == ID_CALLBACK_END:
break
self._do_next(task_id)
queue.task_done()
self._idle_checking()
queue.task_done()
def _border_check(self, task_id):
if task_id >= len(
self._full_queue) or self._full_queue[task_id] is None:
raise IndexError('task id is invalid.')
def _open(self, task_id):
wrapper = None
dlreq_wrapper = self._full_queue[task_id]
handler = self._open_handle(child_process=dlreq_wrapper.child_process,
**self._open_kwargs)
if dlreq_wrapper.child_process:
self._process_collector[task_id] = handler._process
open_callback = dlreq_wrapper.callback
self._thread_collector.put(self._opening_control_thread,
GROUP_OPEN_CTRL,
args=(task_id, handler),
owner=task_id).start()
try:
handler.open(dlreq_wrapper.source)
except (HandlerError, URLError, ClientError) as e:
open_callback.put_nowait(
SIGNAL_TASK_FAIL(Signal(id=task_id, content=e)))
except Exception as e:
open_callback.put_nowait(
SIGNAL_TASK_FAIL(Signal(id=task_id, content=e)))
raise
else:
if handler.is_opened():
if dlreq_wrapper.child_process:
queue = self._process_queue
else:
queue = self._thread_queue
wrapper = HandlerWrapper(
id=task_id,
source=handler,
child_process=dlreq_wrapper.child_process)
handler.install_external_callback_queue(
SignalQueue(id=task_id, queue=queue))
wrapper.move_from(dlreq_wrapper)
open_callback.put_nowait(SIGNAL_TASK_OPEN(content=wrapper))
else:
open_callback.put_nowait(SIGNAL_TASK_PAUSE(Signal(id=task_id)))
return wrapper
def _pause_opening(self, task_id):
flag = False
open_threads = self._thread_collector.get_group(GROUP_OPEN_CTRL)
for i in open_threads:
if i.owner == task_id:
flag = True
if flag:
wrapper = self._full_queue[task_id]
if type(wrapper) is RequestWrapper:
wrapper.callback.put_nowait(SIGNAL_TASK_PAUSE())
def _opened_info_checking(self, task_id):
return type(self._full_queue[task_id]) is not RequestWrapper
def _send_ctrl_signal(self, signal_wrapper, task_id, content=None):
signal = signal_wrapper(Signal(id=task_id, content=content))
if self._full_queue[task_id].child_process:
self._process_queue.put_nowait(signal)
else:
self._thread_queue.put_nowait(signal)
def _close_controller(self):
if self._thread_collector.get_group(GROUP_CTRL_T):
self._thread_queue.put_nowait(SIGNAL_CALLBACK_END)
if self._thread_collector.get_group(GROUP_CTRL_P):
self._process_queue.put_nowait(SIGNAL_CALLBACK_END)
def _idle_checking(self):
if not self._queue.opening and not self._queue.running and \
(not self._queue_started or (self._queue_started and not self._queue.queue)):
self._close_controller()
def _do_next(self, task_id):
next_task = self._full_queue[task_id].get_next()
if next_task:
self._thread_collector.put(next_task.handler,
GROUP_DO_NEXT,
args=next_task.args,
kwargs=next_task.kwargs,
owner=task_id).start()
def __remove_queue_task(self, task_id):
if task_id in self._queue.running:
self._queue.running.remove(task_id)
elif task_id in self._queue.queue:
self._queue.queue.remove(task_id)
elif task_id in self._queue.paused:
self._queue.paused.remove(task_id)
elif task_id in self._queue.ready:
self._queue.ready.remove(task_id)
elif task_id in self._queue.unopened:
self._queue.unopened.remove(task_id)
elif task_id in self._queue.failed:
self._queue.failed.remove(task_id)
elif task_id in self._queue.finished:
self._queue.finished.remove(task_id)
elif task_id in self._queue.opening:
self._queue.opening.remove(task_id)
def __paused_task_handler(self, signal):
task_id = signal.id
if task_id in self._queue.running:
self._full_queue[task_id].source.pause(block=False)
elif task_id in self._queue.unopened:
self._pause_opening(task_id)
if task_id in self._queue.running:
self._queue.running.remove(task_id)
self._queue.paused.append(task_id)
if self._queue_started:
self.start_queue()
def __check_process_collector(self, task_id):
if task_id in self._process_collector:
del self._process_collector[task_id]
def __opened_task_handler(self, task_id):
self.__remove_queue_task(task_id)
self._queue.ready.append(task_id)
def __finished_task_handler(self, task_id):
self.__remove_queue_task(task_id)
self._queue.finished.append(task_id)
self._full_queue[task_id].source.close()
self.__check_process_collector(task_id)
if self._queue_started:
self.start_queue()
def __failed_task_handler(self, task_id, exception):
self.__remove_queue_task(task_id)
self._full_queue[task_id].fail(exception)
self._queue.failed.append(task_id)
self._full_queue[task_id].reset()
self.__check_process_collector(task_id)
if self._queue_started:
self.start_queue()
def __stopped_task_handler(self, task_id):
pass
def __started_task_handler(self, task_id):
self.__remove_queue_task(task_id)
self._queue.running.append(task_id)
class BasicSpider:
def __init__(self, max_task, url_page_format, max_page=1000, start_page=1):
self.next_page = start_page
self.start_page = start_page
self.max_task = max_task
self.max_page = max_page
self._queue = Queue()
self.url_page_format = url_page_format
self.page_number_data = [None for _ in range(max_page)]
def _task_controller(self):
task_done_counter = 0
while True:
page, _ = self._queue.get()
self._queue.task_done()
if page is not None:
task_done_counter += 1
print(page, task_done_counter)
if task_done_counter % 50 == 0:
self.save_addup_data('./data/%s-%s.txt' %
(__name__, time.time()))
if task_done_counter + self.start_page - 1 >= self.max_page:
self.save_addup_data('./data/%s-%s.txt' %
(__name__, time.time()))
print(self.get_total())
break
else:
if self.next_page <= self.max_page:
print('NEXT_PAGE: %d' % self.next_page)
self._run_next_page()
def _spider_handler(self, page, **kwargs):
while not self._addup_wrapper(page, **kwargs):
pass
self._queue.put_nowait((page, kwargs))
def _run_next_page(self, kwargs=None):
if self.next_page > self.max_page:
return
while self.page_number_data[self.next_page - 1] is not None:
self.next_page += 1
if self.next_page > self.max_page:
self._queue.put_nowait((None, kwargs or {}))
return
threading.Thread(target=self._spider_handler,
args=(self.next_page, ),
kwargs=kwargs).start()
self.next_page += 1
def run(self):
for i in range(self.max_task):
self._run_next_page()
time.sleep(0.1)
ctrl = threading.Thread(target=self._task_controller,
name='task_controller')
ctrl.start()
# for inspection
while True:
time.sleep(1)
def _addup_wrapper(self, page, **kwargs):
try:
self.get_page_addup(page, **kwargs)
except Exception as e:
print('===========error')
print(page)
print(e)
# print_exc()
print('===========error')
else:
return True
return False
def save_addup_data(self, name):
tl = [str(i) for i in self.page_number_data]
with open(name, 'w') as f:
f.write('\n'.join(tl))
def get_bs4(self, page):
req = requests.get(self.url_page_format % page, headers=HEADERS)
text = req.text
bs4 = BeautifulSoup(text, 'lxml')
return bs4
def get_page_addup(self, page, **kwargs):
return
def get_total(self):
if list(self.check_page_number()):
raise ValueError('addup data is incomplete ')
counter = 0
for i in self.page_number_data:
counter += i
return counter
def check_page_number(self):
for i, v in enumerate(self.page_number_data):
if v is None:
yield i
def load_page_data(self, filepath):
self.page_number_data = []
with open(filepath, 'r') as f:
data_str = f.read()
for i in data_str.split('\n'):
if i:
if i.strip() == 'None' or i.strip() == '0':
self.page_number_data.append(None)
else:
self.page_number_data.append(int(i.strip()))
class TilestorWriter:
__instance = None
def __init__(self):
if TilestorWriter.__instance is None:
self.queue = Queue(maxsize=QUEUE_MAX_SIZE)
self.cstart = None
self._worker = Thread(target=self._job)
self._worker.daemon = True
self._worker.start()
@classmethod
def getInstance(cls):
if cls.__instance is None:
cls.__instance = TilestorWriter()
return cls.__instance
def put(self, payload):
cstart = self.cstart
if cstart is not None:
cdelta = time() - cstart
if cdelta > QUEUE_STUCK_TIMEOUT:
raise TileWriterQueueStuckException(
"Tile writer queue is stuck for {} seconds.".format(
cdelta))
try:
self.queue.put_nowait(payload)
except Full:
raise TileWriterQueueFullException(
"Tile writer queue is full at maxsize {}.".format(
self.queue.maxsize))
def _job(self):
while True:
self.cstart = None
data = self.queue.get()
self.cstart = time()
# Tile cache writer may fall sometimes in case of database connecti
# problem for example. So we just skip a tile with error and log an
# exception.
try:
z, x, y = data['tile']
tstamp = int(
(datetime.utcnow() - TIMESTAMP_EPOCH).total_seconds())
img = data['img']
colortuple = imgcolor(img)
color = pack_color(
colortuple) if colortuple is not None else None
with transaction.manager:
conn = DBSession.connection()
conn.execute(db.sql.text(
'DELETE FROM tile_cache."{0}" WHERE z = :z AND x = :x AND y = :y; '
'INSERT INTO tile_cache."{0}" (z, x, y, color, tstamp) '
'VALUES (:z, :x, :y, :color, :tstamp)'.format(
data['uuid'])),
z=z,
x=x,
y=y,
color=color,
tstamp=tstamp)
# Force zope session management to commit changes
mark_changed(DBSession())
if color is None:
buf = BytesIO()
img.save(buf, format='PNG', compress_level=3)
tilestor = get_tile_db(data['db_path'])
tilestor.execute(
"DELETE FROM tile WHERE z = ? AND x = ? AND y = ?",
(z, x, y))
try:
tilestor.execute(
"INSERT INTO tile VALUES (?, ?, ?, ?, ?)",
(z, x, y, tstamp, buf.getvalue()))
except sqlite3.IntegrityError:
# NOTE: Race condition with other proccess may occurs here.
# TODO: ON CONFLICT DO ... in SQLite >= 3.24.0 (python 3)
pass
except Exception as exc:
_logger.exception("Uncaught exception in tile writer: %s",
exc.message)
if 'answer_queue' in data:
data['answer_queue'].put_nowait(None)
class TCPHandler(logging.Handler):
'''
Python logging handler for sending JSON formatted messages over
TCP, optionally wrapping the connection with TLSv1
'''
def __init__(self, host, port, ssl_ca_file=None):
'''
Instantiate a TCPHandler with the intent of connecting to the
given host (string) and port (int) with or without using SSL/TLSv1
'''
logging.Handler.__init__(self)
self.host = host
self.port = port
self.ssl_ca_file = ssl_ca_file
self.sock = None
self.queue = Queue(LOG_QUEUE_SIZE)
self.connect_wait = BACKOFF_INITIAL
self.raiseExceptions = 0
self.hostname = socket.gethostname()
if self.hostname.find('.') != -1:
self.hostname = self.hostname.split('.', 1)[0]
self.sender = threading.Thread(target=self.run)
self.sender.setDaemon(True)
self.sender.start()
def connect(self):
'''
Create a connection with the server, sleeping for some
period of time if connection errors have occurred recently.
'''
self.sock = socket.socket()
if self.ssl_ca_file:
self.sock = ssl.wrap_socket(self.sock,
ssl_version=ssl.PROTOCOL_TLSv1,
cert_reqs=ssl.CERT_REQUIRED,
ca_certs=self.ssl_ca_file)
INTERNAL_LOG.debug('Connecting (backoff: %.03f)' %
self.connect_wait)
time.sleep(self.connect_wait)
self.sock.connect((self.host, self.port))
def jsonify(self, record):
'''
Translate a LogRecord instance into a json_event
'''
timestamp = datetime.utcfromtimestamp(record.created)
timestamp = timestamp.isoformat()
fields = {
'level': record.levelname,
'filename': record.pathname,
'lineno': record.lineno,
'method': record.funcName,
}
if record.exc_info:
fields['exception'] = str(record.exc_info)
fields['traceback'] = format_exc(record.exc_info)
log = {
'@source_host': self.hostname,
'@timestamp': timestamp,
'@tags': [record.name],
'@message': record.getMessage(),
'@fields': fields,
}
return json.dumps(log)
def emit(self, record):
'''
Send a LogRecord object formatted as json_event via a
queue and worker thread.
'''
self.queue.put_nowait(record)
def run(self):
'''
Main loop of the logger thread. All network I/O and exception handling
originates here. Strings are consumed from self.queue and sent to
self.sock, creating a new connection if necessary.
If any exceptions are caught, the message is put() back on the queue
and the exception is allowed to propagate up through
logging.Handler.handleError(), potentially causing this thread to abort.
'''
INTERNAL_LOG.debug('Log I/O thread started')
while True:
record = self.queue.get()
if record is None:
break
jsonrecord = self.jsonify(record)
jsonrecord = '%s\n' % jsonrecord
try:
if self.sock is None:
self.connect()
self.send(jsonrecord)
except Exception:
# This exception will be silently ignored and the message
# requeued unless self.raiseExceptions=1
self.queue.put(record)
self.handleError(record)
self.queue.task_done()
INTERNAL_LOG.debug('Log I/O thread exited cleanly')
def send(self, data):
'''
Keep calling SSLSocket.write until the entire message has been sent
'''
while len(data) > 0:
if self.ssl_ca_file:
sent = self.sock.write(data)
else:
sent = self.sock.send(data)
data = data[sent:]
self.connect_wait = BACKOFF_INITIAL
def handleError(self, record):
'''
If an error occurs trying to send the log message, close the connection
and delegate the exception handling to the superclass' handleError,
which raises the exception (potentially killing the log thread) unless
self.raiseExceptions is False.
http://hg.python.org/cpython/file/e64d4518b23c/Lib/logging/__init__.py#l797
'''
INTERNAL_LOG.exception('Unable to send log')
self.cleanup()
self.connect_wait *= BACKOFF_MULTIPLE
logging.Handler.handleError(self, record)
def cleanup(self):
'''
If the socket to the server is still open, close it. Otherwise, do
nothing.
'''
if self.sock:
INTERNAL_LOG.info('Closing socket')
self.sock.close()
self.sock = None
def close(self):
'''
Send a sentinel None object to the worker thread, telling it to exit
and disconnect from the server.
'''
self.queue.put(None)
self.cleanup()
class KeyboardCapture(threading.Thread):
"""Implementation of KeyboardCapture for OSX."""
_KEYBOARD_EVENTS = set([kCGEventKeyDown, kCGEventKeyUp])
def __init__(self):
threading.Thread.__init__(self, name="KeyboardEventTapThread")
self._loop = None
self._event_queue = Queue() # Drained by event handler thread.
self._suppressed_keys = set()
self.key_down = lambda key: None
self.key_up = lambda key: None
# Returning the event means that it is passed on
# for further processing by others.
#
# Returning None means that the event is intercepted.
#
# Delaying too long in returning appears to cause the
# system to ignore the tap forever after
# (https://github.com/openstenoproject/plover/issues/484#issuecomment-214743466).
#
# This motivates pushing callbacks to the other side
# of a queue of received events, so that we can return
# from this callback as soon as possible.
def callback(proxy, event_type, event, reference):
SUPPRESS_EVENT = None
PASS_EVENT_THROUGH = event
# Don't pass on meta events meant for this event tap.
is_unexpected_event = event_type not in self._KEYBOARD_EVENTS
if is_unexpected_event:
if event_type == kCGEventTapDisabledByTimeout:
# Re-enable the tap and hope we act faster next time
CGEventTapEnable(self._tap, True)
plover.log.warning("Keystrokes may have been missed. " + "Keyboard event tap has been re-enabled. ")
return SUPPRESS_EVENT
# Don't intercept the event if it has modifiers, allow
# Fn and Numeric flags so we can suppress the arrow and
# extended (home, end, etc...) keys.
suppressible_modifiers = (
kCGEventFlagMaskNumericPad | kCGEventFlagMaskSecondaryFn | kCGEventFlagMaskNonCoalesced
)
has_nonsupressible_modifiers = CGEventGetFlags(event) & ~suppressible_modifiers
if has_nonsupressible_modifiers and event_type == kCGEventKeyDown:
return PASS_EVENT_THROUGH
keycode = CGEventGetIntegerValueField(event, kCGKeyboardEventKeycode)
key = KEYCODE_TO_KEY.get(keycode)
self._async_dispatch(key, event_type)
if key in self._suppressed_keys:
return SUPPRESS_EVENT
return PASS_EVENT_THROUGH
self._tap = CGEventTapCreate(
kCGSessionEventTap,
kCGHeadInsertEventTap,
kCGEventTapOptionDefault,
CGEventMaskBit(kCGEventKeyDown) | CGEventMaskBit(kCGEventKeyUp),
callback,
None,
)
if self._tap is None:
# Todo(hesky): See if there is a nice way to show the user what's
# needed (or do it for them).
raise Exception("Enable access for assistive devices.")
CGEventTapEnable(self._tap, False)
def run(self):
source = CFMachPortCreateRunLoopSource(None, self._tap, 0)
handler_thread = threading.Thread(target=self._event_handler, name="KeyEventDispatcher")
handler_thread.start()
self._loop = CFRunLoopGetCurrent()
CFRunLoopAddSource(self._loop, source, kCFRunLoopCommonModes)
CGEventTapEnable(self._tap, True)
CFRunLoopRun()
# Wake up event handler.
self._event_queue.put_nowait(None)
handler_thread.join()
CFMachPortInvalidate(self._tap)
CFRelease(self._tap)
CFRunLoopSourceInvalidate(source)
def cancel(self):
CFRunLoopStop(self._loop)
self.join()
self._loop = None
def suppress_keyboard(self, suppressed_keys=()):
self._suppressed_keys = set(suppressed_keys)
def _async_dispatch(self, key, event_type):
"""
Dispatches a key string in KEYCODE_TO_KEY.values() and a CGEventType
to the appropriate KeyboardCapture callback
without blocking execution of its caller.
"""
if key is None:
return
is_keyup = event_type == kCGEventKeyUp
pair = (key, is_keyup)
self._event_queue.put_nowait(pair)
def _event_handler(self):
"""
Event dispatching thread launched during run().
Loops until None is received from _event_queue.
Avoids busy-waiting by blocking on _event_queue.
In normal operation, it gets a pair of
(key_string, is_keyup_bool) from _event_queue
and routes the string to self.key_up or self.key_down,
then waits for a new pair to arrive.
"""
while True:
pair = self._event_queue.get(block=True, timeout=None)
if pair is None:
return
key, is_keyup = pair
handler = self.key_up if is_keyup else self.key_down
handler(key)
class eptWorkerFabric(object):
""" tracks cache and settings for each fabric actively being monitored, also provides useful
notification and push_event functions
"""
def __init__(self, fabric):
self.fabric = fabric
self.start_ts = time.time()
self.settings = eptSettings.load(fabric=fabric, settings="default")
self.cache = eptCache(fabric)
self.dns_cache = DNSCache()
self.db = get_db()
self.watcher_paused = False
self.session = None
self.notify_queue = None
self.notify_thread = None
self.init()
def init(self):
""" initialize settings after fabric settings as been loaded """
# epm parser used with eptWorker for creating pseudo eevents
self.ept_epm_parser = eptEpmEventParser(self.fabric,
self.settings.overlay_vnid)
# one time calculation for email address and syslog server (which requires valid port)
self.email_address = self.settings.email_address
self.syslog_server = self.settings.syslog_server
self.syslog_port = self.settings.syslog_port
if len(self.email_address) == 0:
self.email_address = None
if len(self.syslog_server) == 0:
self.syslog_server = None
self.syslog_port = None
def close(self):
""" stateful close when worker receives FABRIC_STOP for this fabric """
if self.db is not None:
self.db.client.close()
if self.session is not None:
self.session.close()
if self.notify_thread is not None:
self.notify_thread.exit()
# remove all objects from notify queue
if self.notify_queue is not None:
try:
logger.debug("clearing notify queue (size: %d)",
self.notify_queue.qsize())
while not self.notify_queue.empty():
self.notify_queue.get()
except Exception as e:
logger.debug("Traceback:\n%s", traceback.format_exc())
logger.error("failed to execute clear notify queue %s", e)
def watcher_init(self):
""" watcher process needs session object for mo sync and notify queue"""
logger.debug("wf worker init for %s", self.fabric)
self.notify_queue = Queue(maxsize=NOTIFY_QUEUE_MAX_SIZE)
logger.debug("starting worker fabric apic session")
self.session = get_apic_session(self.fabric)
if self.session is None:
logger.error("failed to get session object within worker fabric")
# watcher will also send notifications within a background thread to ensure that
# any delay in syslog or email does not backup other service events
self.notify_thread = BackgroundThread(func=self.execute_notify,
name="notify",
count=0,
interval=NOTIFY_INTERVAL)
self.notify_thread.daemon = True
self.notify_thread.start()
def settings_reload(self):
""" reload settings from db """
logger.debug("reloading settings for %s", self.fabric)
self.settings.reload()
self.init()
def get_uptime_delta_offset(self, delta=None):
""" return difference between provided delta and current uptime. If the uptime_delta is
less than zero, return 0. If no delta is provided, then return the uptime.
"""
uptime = time.time() - self.start_ts
if delta is None: return uptime
uptime_delta = delta - uptime
if uptime_delta > 0: return uptime_delta
return 0
def push_event(self, table, key, event, per_node=True):
# wrapper to push an event to eptHistory events list. set per_node to false to use
# max_endpoint_event rotate length, else max_per_node_endpoint_events value is used
if per_node:
return push_event(
self.db[table],
key,
event,
rotate=self.settings.max_per_node_endpoint_events)
else:
return push_event(self.db[table],
key,
event,
rotate=self.settings.max_endpoint_events)
def get_learn_type(self, vnid, flags=[]):
# based on provide vnid and flags return learn type for endpoint:
# loopback - if loopback in flags
# psvi - if psvi in flags
# svi - represents an external SVI (should be set on all external svis)
# overlay if vnid is overlay vnid
# external if vnid is in eptVnid table with external set to true
# else returns 'epg' (default learn type)
if "loopback" in flags:
return "loopback"
elif "psvi" in flags:
return "psvi"
elif "svi" in flags:
return "external"
elif vnid == self.settings.overlay_vnid:
return "overlay"
ept_vnid = self.cache.get_vnid_name(vnid, return_object=True)
if ept_vnid is not None and ept_vnid.external:
return "external"
return "epg"
def notification_enabled(self, notify_type):
# return dict with email address, syslog server, syslog port for notify type. If not enabled,
# then return None for each field. Set notify_type to 'any_email' or 'any_syslog' to force
# test of a particular notification type
ret = {
"enabled": False,
"email_address": None,
"syslog_server": None,
"syslog_port": None
}
if notify_type == "move":
attr = ("notify_move_email", "notify_move_syslog")
elif notify_type == "stale":
attr = ("notify_stale_email", "notify_stale_syslog")
elif notify_type == "offsubnet":
attr = ("notify_offsubnet_email", "notify_offsubnet_syslog")
elif notify_type == "clear":
attr = ("notify_clear_email", "notify_clear_syslog")
elif notify_type == "rapid":
attr = ("notify_rapid_email", "notify_rapid_syslog")
elif notify_type == "any_email":
# return all notification types enabled
ret["enabled"] = True
ret["email_address"] = self.email_address
return ret
elif notify_type == "any_syslog":
# return all notification types enabled
ret["enabled"] = True
ret["syslog_server"] = self.syslog_server
ret["syslog_port"] = self.syslog_port
return ret
else:
logger.warn("invalid notification type '%s", notify_type)
return ret
if getattr(self.settings, attr[0]):
ret["enabled"] = True
ret["email_address"] = self.email_address
if getattr(self.settings, attr[1]):
ret["enabled"] = True
ret["syslog_server"] = self.syslog_server
ret["syslog_port"] = self.syslog_port
return ret
def send_notification(self,
notify_type,
subject=None,
txt=None,
bulk=None):
# send proper notifications for this fabric. set notify_type to none to skip enable check
# and force notification. user can set bulk to list of (subject/txt) tuples to send a list
# of notifications at the same time. All notifications must be of the same notify_type.
success = True
errmsg = ""
notify = self.notification_enabled(notify_type)
if notify["enabled"]:
if notify["email_address"] is not None:
emails = []
if bulk is not None:
for (bulk_subject, bulk_txt) in bulk:
emails.append({
"sender":
get_app_config().get("EMAIL_SENDER", None),
"receiver":
notify["email_address"],
"subject":
bulk_subject,
"msg":
bulk_txt,
})
else:
emails.append({
"sender":
get_app_config().get("EMAIL_SENDER", None),
"receiver":
notify["email_address"],
"subject":
subject,
"msg":
txt,
})
# send_email already supports a list of emails, so simply send all at once
(success, errmsg) = send_emails(settings=self.settings,
dns_cache=self.dns_cache,
emails=emails)
if not success:
logger.warn("failed to send email: %s", errmsg)
if notify["syslog_server"] is not None:
if bulk is not None:
for (bulk_subject, bulk_txt) in bulk:
syslog(
bulk_txt,
dns_cache=self.dns_cache,
server=notify["syslog_server"],
server_port=notify["syslog_port"],
)
else:
syslog(
txt,
dns_cache=self.dns_cache,
server=notify["syslog_server"],
server_port=notify["syslog_port"],
)
return (success, errmsg)
else:
logger.debug("skipping send notification as '%s' is not enabled",
notify_type)
return (False, "notification not enabled")
def queue_notification(self, notify_type, subject, txt):
# queue notification that will be sent at next iteration of NOTIFY_INTERVAL
if self.notify_queue is None:
logger.error("notify queue not initialized for worker fabric")
return
try:
logger.debug("enqueuing %s notification (queue size %d)",
notify_type, self.notify_queue.qsize())
self.notify_queue.put_nowait(
(notify_type, subject, txt, time.time()))
except Full as e:
logger.error(
"failed to enqueue notification, queue is full (size: %s)",
self.notify_queue.qsize())
def execute_notify(self):
# send any notifications sitting in queue, log number of notifications sent and max queue
# time. We want to support bulk notifications (mainly for email to prevent multiple login
# on smpt_relay_auth setups) so this function will sort based on notify type and execute
# send notification with bulk flag.
msgs = {} # indexed by notify type and contains tuple (subject,txt)
count = 0
max_queue_time = 0
while not self.notify_queue.empty():
(notify_type, subject, txt, q_ts) = self.notify_queue.get()
count += 1
q_time = time.time() - q_ts
if q_time > max_queue_time:
max_queue_time = q_time
if notify_type not in msgs:
msgs[notify_type] = []
msgs[notify_type].append((subject, txt))
for notify_type in msgs:
self.send_notification(notify_type, bulk=msgs[notify_type])
if count > 0:
logger.debug("sent %s notifications, max queue time %0.3f sec",
count, max_queue_time)
class QueueWorker(BaseWorker):
TIMEOUT_ATTEMPTS = 5
QUEUE_SIZE = -1 # inf
END_EVENT = object()
NAME = 'polyaxon.QueueWorker'
def __init__(self, timeout=None, queue_size=None):
super(QueueWorker, self).__init__()
self._queue = Queue(queue_size or self.QUEUE_SIZE)
self._timeout = timeout if timeout is not None else settings.TIMEOUT
def atexit(self):
with self._lock:
if not self.is_alive():
return
self._queue.put_nowait(self.END_EVENT)
def timeout_join(timeout, queue):
end = time() + timeout
queue.all_tasks_done.acquire()
try:
while queue.unfinished_tasks:
current_timeout = end - time()
if current_timeout <= 0:
# timed out
return False
queue.all_tasks_done.wait(timeout=current_timeout)
return True
finally:
queue.all_tasks_done.release()
# ensure wait
timeout = min(settings.MIN_TIMEOUT,
self._timeout / self.TIMEOUT_ATTEMPTS)
if timeout_join(timeout=timeout, queue=self._queue):
timeout = 0
else:
# Queue still has message, try another time
size = self._queue.qsize()
if not settings.IN_CLUSTER:
print(
'Polyaxon %s is attempting to send %i pending messages'
% (self.NAME, size))
print('Waiting up to {} seconds'.format(self._timeout))
if os.name == 'nt':
print('Press Ctrl-Break to quit')
else:
print('Press Ctrl-C to quit')
sleep(settings.MIN_TIMEOUT) # Allow tasks to get executed
while timeout > 0 and not timeout_join(timeout=timeout,
queue=self._queue):
timeout = min(timeout + self._timeout / self.TIMEOUT_ATTEMPTS,
self._timeout - timeout)
size = self._queue.qsize()
if size > 0:
print(
'Polyaxon %s timed out and did not manage to send %i messages'
% (self.NAME, size))
self._thread = None
def stop(self, timeout=None):
with self._lock:
if self._thread:
self._queue.put_nowait(self.END_EVENT)
self._thread.join(timeout=timeout)
self._thread = None
self._thread_for_pid = None
def queue(self, callback, *args, **kwargs):
self.is_running()
self._queue.put_nowait((callback, args, kwargs))
def _target(self):
while True:
record = self._queue.get()
try:
if record is self.END_EVENT:
break
callback, args, kwargs = record
try:
callback(*args, **kwargs)
except Exception:
logger.error('Failed processing job', exc_info=True)
finally:
self._queue.task_done()
sleep(0)
class SocketServer(object):
"""ZMQ-based socket server for sending and receiving messages from the host
PC.
Because of the weird way in which PyEPL handles events, we can't run this as
its own thread, but instead have to poll for events in the general PyEPL
machinery. In the future, we should clean up PyEPL entirely so that it does
not block other threads (amongst other reasons).
:param zmq.Context ctx:
"""
def __init__(self, ctx=None):
self.ctx = ctx or zmq.Context()
self._handlers = []
self.sock = self.ctx.socket(zmq.PAIR)
self._bound = False
self.poller = zmq.Poller()
self.poller.register(self.sock, zmq.POLLIN)
# Outgoing message queue
self._out_queue = Queue()
# time of last sent heartbeat message
self._last_heartbeat = 0.
# Logging of sent and received messages.
self.logger = create_logger("network")
def join(self):
"""Block until all outgoing messages have been processed."""
self.logger.warning("Joining doesn't work yet; doing nothing...")
# self._out_queue.join()
def bind(self, address="tcp://*:8889"):
"""Bind the socket to start listening for connections.
:param str address: ZMQ address string
"""
self.sock.bind(address)
self._bound = True
def register_handler(self, func):
"""Register a message handler.
:param callable func: Handler function which takes the message as its
only argument.
"""
self.logger.debug("Adding handler: %s", func.__name__)
self._handlers.append(func)
def enqueue_message(self, msg):
"""Submit a new outgoing message to the queue."""
self._out_queue.put_nowait(msg)
def send(self, msg):
"""Immediately transmit a message to the host PC. It is advisable to not
call this method directly in most cases, but rather enqueue a message to
be sent via :meth:`enqueue_message`.
:param RAMMessage msg: Message to send.
"""
out = msg.jsonize()
try:
self.log_message(msg, incoming=False)
self.sock.send(out, zmq.NOBLOCK)
except:
self.logger.error("Sending failed!")
def send_heartbeat(self):
"""Convenience method to send a heartbeat message to the host PC."""
if time.time() - self._last_heartbeat >= 1.0:
self.send(HeartbeatMessage())
self._last_heartbeat = time.time()
def log_message(self, message, incoming=True):
"""Log a message to the log file."""
if not incoming:
message = message.to_dict()
message["in_or_out"] = "in" if incoming else "out"
self.logger.info("%s", json.dumps(message))
def handle_incoming(self):
events = self.poller.poll(1)
if self.sock in dict(events):
try:
msg = self.sock.recv_json()
self.log_message(msg, incoming=True)
except:
self.logger.error("Unable to decode JSON.", exc_info=True)
return
for handler in self._handlers:
try:
handler(msg)
except:
self.logger.error("Error handling message", exc_info=True)
continue
def handle_outgoing(self):
try:
while not self._out_queue.empty():
msg = self._out_queue.get_nowait()
self.send(msg)
self._out_queue.task_done(
) # so we can join the queue elsewhere
except:
self.logger.error("Error in outgoing message processing",
exc_info=True)
def update(self):
"""Call periodically to check for incoming messages and/or send messages
in the outgoing queue.
"""
self.handle_incoming()
self.handle_outgoing()
class MultiClientConsole(AbstractConsole):
def __init__(self, file, url, buffsize, block_size, maxthread,
exception_callback):
self._blocks = []
self._block_size = block_size
self._mxthread = maxthread
self._buffsize = buffsize
self.__total = int(ceil(file.getsize() / self._block_size))
self.file = file
self._url = url
self._tpm = ThreadCollector()
self._client_callback_queue = Queue()
self.__exception_signal = exception_callback
self.__buff_counter = 0
self._buff_lock = Lock()
self.runflag = False
self._acum_time = AccumulatedTime(0)
self._inst_maker = InstSpeedMaker()
self._saver = None
self._external_callback_queue = None
def insert(self, content, progress, abs_grid=None, rel_grid=None):
block = Block(content,
progress,
self._block_size,
abs_grid=abs_grid,
rel_grid=rel_grid)
for i, v in enumerate(self._blocks):
if v.begin > block.begin:
self._blocks.insert(i, block)
break
else:
self._blocks.append(block)
return block
def locate(self, block_index):
c = 0
for i in self._blocks:
c += i.length
if block_index < c:
return i
def getfullmap(self):
for b in self._blocks:
for i in b.abs_grid:
yield i
def __iter__(self):
return self.getfullmap()
def __getitem__(self, item):
if type(item) is not int:
raise ValueError()
if item >= self.__total:
raise IndexError()
c = 0
for i in self._blocks:
c += i.length
if item < c:
return i.grid[item - c + i.length]
raise IndexError()
def prepare(self):
self.file.open()
def run(self):
""" Run download task. """
if not self._blocks:
raise RuntimeError(
'cannot find any download block in the console.')
self.runflag = True
self._acum_time.start()
self._inst_maker.start(self.get_go_inc())
for block in self._blocks:
block.clear_signal()
self._tpm.put(block.handler,
GROUP_CLIENT,
args=(self._client_callback_queue, ),
owner=block)
self._tpm.start_group(GROUP_CLIENT)
self._tpm.put(self._client_callback_handle_thread,
GROUP_CONTROLLER).start()
self._tpm.put(self._inst_speed_capture_thread, GROUP_INSTSPEED).start()
if self._mxthread > self.get_client_cnt():
self._make_slice_request()
def get_block_size(self):
""" Return console's unit block size. """
return self._block_size
def get_go_inc(self):
""" Return byte size downloaded. """
inc = 0
for v in self._blocks:
inc += v.get_go_inc()
return inc
def pause(self, block=True, timeout=None):
""" Pause downloading. block until finish pausing only when block == True. """
self.runflag = False
self._client_callback_queue.put_nowait(SIGNAL_TASK_PAUSE())
if block:
self._tpm.join(timeout=timeout)
stop = pause
def getavgspeed(self):
""" Return average download speed. """
return self.get_go_inc() / (self._acum_time.getinctime()
or float('inf'))
def getinstspeed(self):
""" Return instant download speed. """
return self._inst_maker.getspeed()
def _inst_speed_capture_thread(self):
while self.runflag:
self._inst_maker.capture(self.get_go_inc())
freq = INST_SPEED_REFRESH_FREQUENCY
if freq > 0:
sleep(1 / freq)
else:
break
def get_time_left(self):
""" Return remaining time to finish the download task. """
speed = self.getavgspeed()
return self.get_byte_left() / speed if speed else float('inf')
def get_byte_left(self):
""" Return remaining byte to finish the download task. """
return self.file.getsize() - self.get_go_inc()
def get_online_cnt(self):
""" Return the number of running client-threads. """
return len(self._tpm.get_group(GROUP_CLIENT))
def get_client_cnt(self):
""" Return the number of running clients. """
acum = 0
for v in self._blocks:
if not v.is_go_finished():
acum += 1
return acum
def is_go_finished(self):
""" Return True if download finished (finish writing to file is not necessary). """
for v in self._blocks:
if not v.is_go_finished():
return False
return self._check_complete()
def is_finished(self):
""" Return True if download task finished. """
for v in self._blocks:
if not v.is_finished():
return False
return self._check_complete()
def _client_callback_handle_thread(self):
""" A thread handler about callback signal. """
while True:
signal = self._client_callback_queue.get()
if signal.id == ID_TASK_BUFF:
self.__buffer_signal_handler(signal.content)
elif signal.id == ID_TASK_SLICE:
self.__slice_feedback_handler(signal.content)
elif signal.id == ID_THREAD_END:
self.__client_thread_end_handler(signal.content)
elif signal.id == ID_URL_STATUS:
self.__url_status_handler(signal.content)
elif signal.id == ID_TASK_PAUSE:
self.__pause_handler()
elif signal.id == ID_TASK_FINISH:
self.__finish_handler()
elif signal.id == ID_TASK_STOP:
self.__stop_handler()
elif signal.id == ID_CALLBACK_END:
break
self._client_callback_queue.task_done()
self._client_callback_queue.task_done()
def _buffer_release_thread(self):
""" A thread handling writing buffer to file. """
with self._buff_lock:
for v in self._blocks:
v.release_buffer(self.file)
if self._saver:
self._saver.dump()
gc.collect()
def _find_block_from_client(self, client, defalut=None):
""" Find the block"""
for v in self._blocks:
if client in v:
return v
return defalut
def _make_slice_request(self):
""" Make a slice request. """
maxblock = sorted(self._blocks, key=lambda i: i.margin,
reverse=True)[0]
margin_len = maxblock.margin
put_begin = maxblock.begin + (maxblock.length - margin_len) + int(
ceil(margin_len / 2))
put_end = maxblock.end
signal = SIGNAL_TASK_SLICE(
(put_begin * self._block_size, put_end * self._block_size))
maxblock.send_signal(signal)
def _check_complete(self):
""" Return True if total blocks length completely match. """
acum_len = 0
for v in self._blocks:
v.calibration()
acum_len += v.length
return self.__total == acum_len
def __client_thread_end_handler(self, client):
""" Callback handler when signal.id == ID_THREAD_END:
<CLIENT THREAD END OF LIFE>
"""
block = self._find_block_from_client(client)
srcwrapper = self._url.getwrapper(block.getsource())
if self.runflag:
if block.is_go_finished():
if self._mxthread > self.get_client_cnt():
self._make_slice_request()
srcwrapper.disuse()
else:
self._tpm.put(client.run,
GROUP_CLIENT,
args=(self._client_callback_queue, ),
owner=block).start()
if self.is_go_finished():
self._buffer_release_thread()
self._finish_task()
def _finish_task(self):
self._client_callback_queue.put_nowait(SIGNAL_TASK_FINISH())
if self._external_callback_queue:
self._external_callback_queue.queue.put_nowait(
SIGNAL_TASK_FINISH(
Signal(id=self._external_callback_queue.id)))
def _failed_task(self, exception):
self.__exception_signal.put_nowait(SIGNAL_EXCEPTION(exception))
if self._external_callback_queue:
self._external_callback_queue.queue.put_nowait(
SIGNAL_EXCEPTION(
Signal(id=self._external_callback_queue.id,
content=exception)))
def __stop_handler(self):
self._buffer_release_thread()
self.file.close()
self._acum_time.stop()
self._inst_maker.stop()
self.runflag = False
self._client_callback_queue.put_nowait(SIGNAL_CALLBACK_END())
self.__exception_signal.put_nowait(SIGNAL_TASK_STOP())
if self._external_callback_queue:
self._external_callback_queue.queue.put_nowait(
SIGNAL_TASK_STOP(Signal(id=self._external_callback_queue.id)))
def __finish_handler(self):
self._client_callback_queue.put_nowait(SIGNAL_TASK_STOP())
def __slice_feedback_handler(self, slice):
""" Callback handler when signal.id == ID_SLICE:
<SLICE FEEDBACK>
"""
if slice.range:
srcwrapper = self._url.get_min_avl_used()
if not srcwrapper:
srcwrapper = self._url.get_min_used()
prog = Progress(slice.range)
client = slice.client(srcwrapper.get(), prog)
block = self.insert(client, prog)
srcwrapper.use()
if self.runflag:
self._tpm.put(block.handler,
GROUP_CLIENT,
args=(self._client_callback_queue, ),
owner=block).start()
if self._mxthread > self.get_client_cnt():
self._make_slice_request()
def __buffer_signal_handler(self, byte):
""" Callback handler when signal.id == ID_BUFF:
<BUFFER COUNTER>
"""
self._url.clear_counter()
self.__buff_counter += byte
if self.__buff_counter >= self._buffsize:
self.__buff_counter = 0
self._tpm.put(self._buffer_release_thread, GROUP_RELEASE).start()
def __url_status_handler(self, url_status):
""" Callback handler when signal.id == ID_URL_STATUS:
<URL SIGNAL>
"""
client = url_status.content.client
exception = url_status.content.exception
source = client.getsource()
srcwrapper = self._url.getwrapper(source)
block = self._find_block_from_client(client)
try:
if url_status.id == ID_NORMAL:
srcwrapper.clear_counter()
elif url_status.id == ID_TIMEOUT:
srcwrapper.timeout()
elif url_status.id == ID_CRASH:
srcwrapper.crash()
elif url_status.id == ID_UNKNOWN:
srcwrapper.unknown()
elif url_status.id == ID_GAIERROR:
srcwrapper.network_down()
except (URLCrash, URLTimeout, URLUnknownError,
NetworkDisconnected) as e:
srcwrapper.disuse()
nextwrapper = self._url.get_min_avl_used()
if not nextwrapper:
nextwrapper = self._url.get_next(srcwrapper.source.id)
if self._url.is_all_crashed():
self._failed_task(e)
else:
block.send_signal(SIGNAL_WAIT(1))
nextwrapper.use_anyway()
block.send_signal(SIGNAL_SWITCH(nextwrapper.get()))
def __pause_handler(self):
""" Callback handler when signal.id == ID_PAUSE:
<PAUSE SIGNAL>
"""
for thread in self._tpm.get_group(GROUP_CLIENT):
thread.owner.send_signal(SIGNAL_TASK_PAUSE())
self._tpm.wait(GROUP_SLICER)
self._tpm.wait(GROUP_CLIENT)
self._tpm.wait(GROUP_RELEASE)
self._client_callback_queue.put_nowait(SIGNAL_TASK_STOP())
if self._external_callback_queue:
self._external_callback_queue.queue.put_nowait(
SIGNAL_TASK_PAUSE(Signal(id=self._external_callback_queue.id)))
def dump_data(self):
return ConsoleDumpedData(block_data=list(
[tuple(b.dump_data()) for b in self._blocks]),
acum_time=self._acum_time.getinctime())
def load(self, dumped_data):
data = ConsoleDumpedData(*dumped_data)
self._acum_time = AccumulatedTime(data.acum_time)
for i in data.block_data:
b = BlockDumpedData(*i)
p = ProgressDumpedData(*b.progress)
prog = Progress(p.range, p.go_inc, p.done_inc)
srcwrapper = self._url.get(b.url_id)
source = srcwrapper.get()
client = build_client(source, prog)
srcwrapper.use()
self.insert(client,
prog,
rel_grid=list([GridCell(*i) for i in b.rel_grid]))
def install_saver(self, saver):
self._saver = saver
def install_external_callback_queue(self, signal_queue):
self._external_callback_queue = signal_queue
def join(self, timeout=None):
self._tpm.join(timeout)
def getBuffCnter(self):
return self.__buff_counter
class SingleClientConsole(AbstractConsole):
def __init__(self, file, url, buffsize, block_size, maxthread,
exception_callback):
self._block_size = block_size
self._buffsize = buffsize
self._block = None
self.file = file
self._url = url
self._tpm = ThreadCollector()
self._client_callback_queue = Queue()
self.__exception_signal = exception_callback
self.__buff_counter = 0
self._buff_lock = Lock()
self.runflag = False
self._acum_time = AccumulatedTime(0)
self._inst_maker = InstSpeedMaker()
self._saver = None
self.__callback_queue = None
def insert(self, client, progress):
if self._block:
raise RuntimeError(
'cannot insert more than one block into single client console.'
)
self._block = Block(client, progress, self._block_size)
def prepare(self):
self.file.open()
def run(self):
self.runflag = True
self._block.clear()
self._acum_time.start()
self._inst_maker.start(self.get_go_inc())
self._tpm.put(self._block.handler,
GROUP_CLIENT,
args=(self._client_callback_queue, ),
owner=self._block).start()
self._tpm.put(self._client_callback_handle_thread,
GROUP_CONTROLLER).start()
self._tpm.put(self._inst_speed_capture_thread, GROUP_INSTSPEED).start()
def __stop_handler(self):
self.file.close()
self._acum_time.stop()
self._inst_maker.stop()
self.runflag = False
self._client_callback_queue.put_nowait(SIGNAL_CALLBACK_END())
self.__exception_signal.put_nowait(SIGNAL_TASK_STOP())
if self.__callback_queue:
self.__callback_queue.queue.put_nowait(
SIGNAL_TASK_STOP(Signal(id=self.__callback_queue.id)))
def pause(self, block=True, timeout=None):
self.runflag = False
self._client_callback_queue.put_nowait(SIGNAL_TASK_PAUSE())
if block:
self._tpm.wait(GROUP_CONTROLLER, timeout=timeout)
def is_finish_go(self):
return self._block.is_go_finished()
def get_byte_left(self):
if self.is_finish_go():
return 0
else:
return float('inf')
def get_time_left(self):
if self.is_finish_go():
return 0
else:
return float('inf')
def getavgspeed(self):
return self.get_go_inc() / (self._acum_time.getinctime()
or float('inf'))
def get_go_inc(self):
return self._block.get_go_inc()
def getinstspeed(self):
return self._inst_maker.getspeed()
def _inst_speed_capture_thread(self):
while self.runflag:
self._inst_maker.capture(self.get_go_inc())
freq = INST_SPEED_REFRESH_FREQUENCY
if freq > 0:
sleep(1 / freq)
else:
break
def _client_callback_handle_thread(self):
while True:
signal = self._client_callback_queue.get()
if signal.id == ID_TASK_BUFF:
self.__buffer_signal_handler(signal.content)
elif signal.id == ID_THREAD_END:
self.__client_thread_end_handler(signal.content)
elif signal.id == ID_URL_STATUS:
self.__url_status_handler(signal.content)
elif signal.id == ID_TASK_PAUSE:
self.__pause_handler()
elif signal.id == ID_TASK_FINISH:
self.__stop_handler()
elif signal.id == ID_TASK_STOP:
break
self._client_callback_queue.task_done()
self._client_callback_queue.task_done()
def is_finished(self):
return self._block.is_finished()
def _finish_task(self):
self._client_callback_queue.put_nowait(SIGNAL_TASK_FINISH())
if self.__callback_queue:
self.__callback_queue.queue.put_nowait(
SIGNAL_TASK_FINISH(Signal(id=self.__callback_queue.id)))
def __pause_handler(self):
""" Callback handler when signal.id == ID_PAUSE:
<PAUSE SIGNAL>
"""
for thread in self._tpm.get_group(GROUP_CLIENT):
thread.owner.send_signal(SIGNAL_TASK_PAUSE())
self._tpm.wait(GROUP_CLIENT)
self._tpm.wait(GROUP_RELEASE)
self._buffer_release_thread()
self._client_callback_queue.put_nowait(SIGNAL_TASK_STOP())
if self.__callback_queue:
self.__callback_queue.queue.put_nowait(
SIGNAL_TASK_PAUSE(Signal(id=self.__callback_queue.id)))
def __url_status_handler(self, url_status):
""" Callback handler when signal.id == ID_URL_STATUS:
<URL SIGNAL>
"""
client = url_status.content.client
exception = url_status.content.exception
source = client.getsource()
srcwrapper = self._url.getwrapper(source)
try:
if url_status.id == ID_NORMAL:
srcwrapper.clear_counter()
elif url_status.id == ID_TIMEOUT:
srcwrapper.timeout()
elif url_status.id == ID_CRASH:
srcwrapper.crash()
elif url_status.id == ID_UNKNOWN:
srcwrapper.unknown()
elif url_status.id == ID_GAIERROR:
srcwrapper.network_down()
except (URLCrash, URLTimeout, URLUnknownError,
NetworkDisconnected) as e:
self.__exception_signal.put_nowait(SIGNAL_EXCEPTION(e))
def _buffer_release_thread(self):
""" A thread handler about releasing buffer. """
with self._buff_lock:
self._block.release_buffer(self.file)
if self._saver:
self._saver.dump()
gc.collect()
def __buffer_signal_handler(self, byte):
""" Callback handler when signal.id == ID_BUFF:
<BUFFER COUNTER>
"""
self._url.clear_counter()
self.__buff_counter += byte
if self.__buff_counter >= self._buffsize:
self.__buff_counter = 0
self._tpm.put(self._buffer_release_thread, GROUP_RELEASE).start()
def __client_thread_end_handler(self, client):
""" Callback handler when signal.id == ID_THREAD_END:
<CLIENT THREAD END OF LIFE>
"""
if self.runflag:
if self._block.is_go_finished():
srcwrapper = self._url.getwrapper(self._block.getsource())
srcwrapper.disuse()
else:
self._block.clear()
self._tpm.put(client.run,
GROUP_CLIENT,
args=(self._client_callback_queue, ),
owner=self._block).start()
if self.is_finish_go():
self._buffer_release_thread()
self._finish_task()
def install_saver(self, saver):
self._saver = saver
def dump_data(self):
return ConsoleDumpedData(block_data=tuple(self._block.dump_data()),
acum_time=self._acum_time.getinctime())
def load(self, dumped_data):
data = ConsoleDumpedData(*dumped_data)
self._acum_time = AccumulatedTime(data.acum_time)
for i in data.block_data:
b = BlockDumpedData(*i)
prog = Progress((0, ))
srcwrapper = self._url.get(b.url_id)
source = srcwrapper.get()
client = build_client(source, prog)
srcwrapper.use()
self.insert(client, prog)
def get_block_size(self):
return self._block_size
def join(self, timeout=None):
self._tpm.join(timeout)
def install_external_callback_queue(self, handle):
self.__callback_queue = handle
def getBuffCnter(self):
return self.__buff_counter
class TCPHandler(logging.Handler):
'''
Python logging handler for sending JSON formatted messages over
TCP, optionally wrapping the connection with TLSv1
'''
def __init__(self, host, port, ssl_ca_file=None):
'''
Instantiate a TCPHandler with the intent of connecting to the
given host (string) and port (int) with or without using SSL/TLSv1
'''
logging.Handler.__init__(self)
self.host = host
self.port = port
self.ssl_ca_file = ssl_ca_file
self.sock = None
self.queue = Queue(LOG_QUEUE_SIZE)
self.connect_wait = BACKOFF_INITIAL
self.raiseExceptions = 0
self.hostname = socket.gethostname()
if self.hostname.find('.') != -1:
self.hostname = self.hostname.split('.', 1)[0]
self.sender = threading.Thread(target=self.run)
self.sender.setDaemon(True)
self.sender.start()
def connect(self):
'''
Create a connection with the server, sleeping for some
period of time if connection errors have occurred recently.
'''
self.sock = socket.socket()
if self.ssl_ca_file:
self.sock = ssl.wrap_socket(self.sock,
ssl_version=ssl.PROTOCOL_TLSv1,
cert_reqs=ssl.CERT_REQUIRED,
ca_certs=self.ssl_ca_file)
INTERNAL_LOG.debug('Connecting (backoff: %.03f)' % self.connect_wait)
time.sleep(self.connect_wait)
self.sock.connect((self.host, self.port))
def jsonify(self, record):
'''
Translate a LogRecord instance into a json_event
'''
timestamp = datetime.utcfromtimestamp(record.created)
timestamp = timestamp.isoformat()
fields = {
'level': record.levelname,
'filename': record.pathname,
'lineno': record.lineno,
'method': record.funcName,
}
if record.exc_info:
fields['exception'] = str(record.exc_info)
fields['traceback'] = format_exc(record.exc_info)
log = {
'@source_host': self.hostname,
'@timestamp': timestamp,
'@tags': [record.name],
'@message': record.getMessage(),
'@fields': fields,
}
return json.dumps(log)
def emit(self, record):
'''
Send a LogRecord object formatted as json_event via a
queue and worker thread.
'''
self.queue.put_nowait(record)
def run(self):
'''
Main loop of the logger thread. All network I/O and exception handling
originates here. Strings are consumed from self.queue and sent to
self.sock, creating a new connection if necessary.
If any exceptions are caught, the message is put() back on the queue
and the exception is allowed to propagate up through
logging.Handler.handleError(), potentially causing this thread to abort.
'''
INTERNAL_LOG.debug('Log I/O thread started')
while True:
record = self.queue.get()
if record is None:
break
jsonrecord = self.jsonify(record)
jsonrecord = '%s\n' % jsonrecord
try:
if self.sock is None:
self.connect()
self.send(jsonrecord)
except Exception:
# This exception will be silently ignored and the message
# requeued unless self.raiseExceptions=1
self.queue.put(record)
self.handleError(record)
self.queue.task_done()
INTERNAL_LOG.debug('Log I/O thread exited cleanly')
def send(self, data):
'''
Keep calling SSLSocket.write until the entire message has been sent
'''
while len(data) > 0:
if self.ssl_ca_file:
sent = self.sock.write(data)
else:
sent = self.sock.send(data)
data = data[sent:]
self.connect_wait = BACKOFF_INITIAL
def handleError(self, record):
'''
If an error occurs trying to send the log message, close the connection
and delegate the exception handling to the superclass' handleError,
which raises the exception (potentially killing the log thread) unless
self.raiseExceptions is False.
http://hg.python.org/cpython/file/e64d4518b23c/Lib/logging/__init__.py#l797
'''
INTERNAL_LOG.exception('Unable to send log')
self.cleanup()
self.connect_wait *= BACKOFF_MULTIPLE
logging.Handler.handleError(self, record)
def cleanup(self):
'''
If the socket to the server is still open, close it. Otherwise, do
nothing.
'''
if self.sock:
INTERNAL_LOG.info('Closing socket')
self.sock.close()
self.sock = None
def close(self):
'''
Send a sentinel None object to the worker thread, telling it to exit
and disconnect from the server.
'''
self.queue.put(None)
self.cleanup()
class TilestorWriter:
__instance = None
def __init__(self):
if TilestorWriter.__instance is None:
self.queue = Queue(maxsize=QUEUE_MAX_SIZE)
self.cstart = None
self._worker = Thread(target=self._job)
self._worker.daemon = True
self._worker.start()
@classmethod
def getInstance(cls):
if cls.__instance is None:
cls.__instance = TilestorWriter()
return cls.__instance
def put(self, payload, timeout):
try:
if timeout is None:
self.queue.put_nowait(payload)
else:
self.queue.put(payload, timeout=timeout)
except Full:
raise TileWriterQueueFullException(
"Tile writer queue is full at maxsize {}.".format(
self.queue.maxsize))
def _job(self):
self._shutdown = False
atexit.register(self.wait_for_shutdown)
data = None
while True:
self.cstart = None
if data is None:
try:
# When the thread is shutting down use minimal timeout
# otherwise use a big timeout not to block shutdown
# that may happen.
get_timeout = 0.001 if self._shutdown else min(
SHUTDOWN_TIMEOUT / 5, 2)
data = self.queue.get(True, get_timeout)
except Empty:
if self._shutdown:
_logger.debug(
"Tile cache writer queue is empty now. Exiting!")
break
else:
continue
db_path = data['db_path']
self.cstart = ptime = time()
tiles_written = 0
time_taken = 0.0
answers = []
# Tile cache writer may fall sometimes in case of database connecti
# problem for example. So we just skip a tile with error and log an
# exception.
try:
with transaction.manager:
conn = DBSession.connection()
tilestor = get_tile_db(db_path)
while data is not None and data['db_path'] == db_path:
z, x, y = data['tile']
tstamp = int((datetime.utcnow() -
TIMESTAMP_EPOCH).total_seconds())
img = data['img']
if img is not None and img.mode != 'RGBA':
img = img.convert('RGBA')
colortuple = imgcolor(img)
color = pack_color(
colortuple) if colortuple is not None else None
self._write_tile_meta(
conn, data['uuid'],
dict(z=z, x=x, y=y, color=color, tstamp=tstamp))
if color is None:
buf = BytesIO()
img.save(buf, format='PNG', compress_level=3)
value = buf.getvalue()
self._write_tile_data(tilestor, z, x, y, tstamp,
value)
if 'answer_queue' in data:
answers.append(data['answer_queue'])
tiles_written += 1
ctime = time()
time_taken += ctime - ptime
if tiles_written >= BATCH_MAX_TILES:
# Break the batch
data = None
else:
# Try to get next tile for the batch. Or break
# the batch if there is no tiles left.
if time_taken < BATCH_DEADLINE:
try:
data = self.queue.get(
timeout=(BATCH_DEADLINE - time_taken))
except Empty:
data = None
else:
data = None
# Do not account queue block time
ptime = time()
# Force zope session management to commit changes
mark_changed(DBSession())
tilestor.commit()
time_taken += time() - ptime
_logger.debug(
"%d tiles were written in %0.3f seconds (%0.1f per "
"second, qsize = %d)", tiles_written, time_taken,
tiles_written / time_taken, self.queue.qsize())
# Report about sucess only after transaction commit
for a in answers:
a.put_nowait(None)
except Exception as exc:
_logger.exception("Uncaught exception in tile writer: %s",
exc.message)
data = None
self.cstart = None
tilestor.rollback()
def _write_tile_meta(self, conn, table_uuid, row):
result = conn.execute(
db.sql.text('SELECT true FROM tile_cache."{}" '
'WHERE z = :z AND x = :x AND y = :y '
'LIMIT 1 FOR UPDATE'.format(table_uuid)), **row)
if result.returns_rows:
conn.execute(
db.sql.text('DELETE FROM tile_cache."{0}" '
'WHERE z = :z AND x = :x AND y = :y '
''.format(table_uuid)), **row)
conn.execute(
db.sql.text(
'INSERT INTO tile_cache."{0}" (z, x, y, color, tstamp) '
'VALUES (:z, :x, :y, :color, :tstamp)'.format(table_uuid)),
**row)
def _write_tile_data(self, tilestor, z, x, y, tstamp, value):
tilestor.execute("DELETE FROM tile WHERE z = ? AND x = ? AND y = ?",
(z, x, y))
try:
tilestor.execute("INSERT INTO tile VALUES (?, ?, ?, ?, ?)",
(z, x, y, tstamp, value))
except sqlite3.IntegrityError:
# NOTE: Race condition with other proccess may occurs here.
# TODO: ON CONFLICT DO ... in SQLite >= 3.24.0 (python 3)
pass
def wait_for_shutdown(self, timeout=SHUTDOWN_TIMEOUT):
if not self._worker.is_alive():
return True
_logger.debug(
"Waiting for shutdown of tile cache writer for %d seconds (" +
"qsize = %d)...", timeout, self.queue.qsize())
self._shutdown = True
self._worker.join(timeout)
if self._worker.is_alive():
_logger.warn(
"Tile cache writer is still running. It'll be killed!")
return False
else:
_logger.debug("Tile cache writer has successfully shut down.")
return True
class JsonRpcServer(object):
log = logging.getLogger("jsonrpc.JsonRpcServer")
"""
Creates new JsonrRpcServer by providing a bridge, timeout in seconds
which defining how often we should log connections stats and thread
factory.
"""
def __init__(self, bridge, timeout, cif, threadFactory=None):
self._bridge = bridge
self._cif = cif
self._workQueue = Queue()
self._threadFactory = threadFactory
self._timeout = timeout
self._next_report = monotonic_time() + self._timeout
self._counter = 0
def queueRequest(self, req):
self._workQueue.put_nowait(req)
"""
Aggregates number of requests received by vdsm. Each request from
a batch is added separately. After time defined by timeout we log
number of requests.
"""
def _attempt_log_stats(self):
self._counter += 1
if monotonic_time() > self._next_report:
self.log.info('%s requests processed during %s seconds',
self._counter, self._timeout)
self._next_report += self._timeout
self._counter = 0
def _serveRequest(self, ctx, req):
start_time = monotonic_time()
response = self._handle_request(req, ctx.server_address)
error = getattr(response, "error", None)
if error is None:
response_log = "succeeded"
else:
response_log = "failed (error %s)" % (error.code, )
self.log.info("RPC call %s %s in %.2f seconds", req.method,
response_log,
monotonic_time() - start_time)
if response is not None:
ctx.requestDone(response)
def _handle_request(self, req, server_address):
self._attempt_log_stats()
logLevel = logging.DEBUG
# VDSM should never respond to any request before all information about
# running VMs is recovered, see https://bugzilla.redhat.com/1339291
if not self._cif.ready:
self.log.info("In recovery, ignoring '%s' in bridge with %s",
req.method, req.params)
return JsonRpcResponse(None, exception.RecoveryInProgress(),
req.id)
self.log.log(logLevel, "Calling '%s' in bridge with %s", req.method,
req.params)
try:
method = self._bridge.dispatch(req.method)
except JsonRpcMethodNotFoundError as e:
if req.isNotification():
return None
return JsonRpcResponse(None, e, req.id)
try:
params = req.params
self._bridge.register_server_address(server_address)
if isinstance(req.params, list):
res = method(*params)
else:
res = method(**params)
self._bridge.unregister_server_address()
except JsonRpcError as e:
return JsonRpcResponse(None, e, req.id)
except Exception as e:
self.log.exception("Internal server error")
return JsonRpcResponse(None, JsonRpcInternalError(str(e)), req.id)
else:
res = True if res is None else res
self.log.log(logLevel, "Return '%s' in bridge with %s", req.method,
res)
if isinstance(res, Suppressed):
res = res.value
return JsonRpcResponse(res, None, req.id)
@traceback(on=log.name)
def serve_requests(self):
while True:
obj = self._workQueue.get()
if obj is None:
break
client, server_address, msg = obj
self._parseMessage(client, server_address, msg)
def _parseMessage(self, client, server_address, msg):
ctx = _JsonRpcServeRequestContext(client, server_address)
try:
rawRequests = json.loads(msg)
except:
ctx.addResponse(JsonRpcResponse(None, JsonRpcParseError(), None))
ctx.sendReply()
return
if isinstance(rawRequests, list):
# Empty batch request
if len(rawRequests) == 0:
ctx.addResponse(
JsonRpcResponse(
None,
JsonRpcInvalidRequestError('request batch is empty',
rawRequests), None))
ctx.sendReply()
return
else:
# From this point on we know it's always a list
rawRequests = [rawRequests]
# JSON Parsed handling each request
requests = []
for rawRequest in rawRequests:
try:
req = JsonRpcRequest.fromRawObject(rawRequest)
requests.append(req)
except JsonRpcError as err:
ctx.addResponse(JsonRpcResponse(None, err, None))
except:
ctx.addResponse(
JsonRpcResponse(None, JsonRpcInternalError(), None))
ctx.setRequests(requests)
# No request was built successfully or is only notifications
if ctx.counter == 0:
ctx.sendReply()
for request in requests:
self._runRequest(ctx, request)
def _runRequest(self, ctx, request):
if self._threadFactory is None:
self._serveRequest(ctx, request)
else:
try:
self._threadFactory(partial(self._serveRequest, ctx, request))
except Exception as e:
self.log.exception("could not allocate request thread")
ctx.requestDone(
JsonRpcResponse(None, JsonRpcInternalError(str(e)),
request.id))
def stop(self):
self.log.info("Stopping JsonRPC Server")
self._workQueue.put_nowait(None)
class HTTPClient(AbstractClient):
__connected = False
def __init__(self, source, progress):
self.source = source
self.progress = progress
self._response = None
self._connection = None
self._callback = None
self.__signal = Queue()
def geturl(self):
return self.source.url
def _buffer(self, buff):
if buff:
self.progress.buffer(buff)
self._callback.put_nowait(SIGNAL_TASK_BUFF(len(buff)))
def _build_connection(self):
if self.source.scheme == 'http':
client_handler = HTTPConnection
elif self.source.scheme == 'https':
client_handler = HTTPSConnection
else:
raise ValueError(
'http client got an unexpected url protocol (%s).' %
self.source.scheme)
conn = client_handler(host=self.source.hostname,
port=self.source.port,
timeout=5)
req_range = (self.progress.begin + self.progress.go_inc,
self.progress.end)
path, headers = self.source.http_request_header(req_range)
conn.request('GET', path, '', dict(headers))
return conn
def _conn_response(self, conn):
res = conn.getresponse()
sleep(0.01)
if res.code in (301, 302, 303, 307):
redurl = res.getheader('location', None)
self.source.http_redirect(redurl)
res.close()
conn.sock.shutdown(socket.SHUT_RDWR)
conn.close()
conn = self._build_connection()
return self._conn_response(conn)
elif 400 <= res.code < 500:
raise HTTP4XXError("(%d)URL: %s " % (res.code, self.source.url))
elif res.code not in (200, 206):
self._callback.put_nowait(SIGNAL_URL_STATUS(SIGNAL_UNKNOWN(self)))
raise URLUnknownError()
return conn, res
def connect(self):
conn = self._build_connection()
conn, res = self._conn_response(conn)
self._connection = conn
self._response = res
self.source.response(self.source.url, res.getheaders(), res.getcode(),
res.length)
self.__connected = True
return res
def retrieve(self):
self.progress.start()
self._callback.put_nowait(
SIGNAL_URL_STATUS(
SIGNAL_NORMAL(ClientException(client=self, exception=None))))
buff = b''
while True:
if not self.__signal.empty():
signal = self.__signal.get()
if signal.id == ID_TASK_SLICE:
putrange = self.progress.slice_check(signal.content)
self._callback.put_nowait(
SIGNAL_TASK_SLICE(
Slice(client=HTTPClient, range=putrange)))
elif signal.id == ID_TASK_PAUSE:
self._buffer(buff)
self.__signal.task_done()
break
elif signal.id == ID_WAIT:
sleep(signal.content)
self.__signal.task_done()
prv_len = len(buff)
remain = self.progress.length - self.progress.go_inc
try:
if remain >= 8192:
buff += self._response.read(8192)
elif remain > 0:
buff += self._response.read(remain)
else:
self._buffer(buff)
break
except (socket.gaierror, URLError, HTTPError, socket.timeout,
Exception) as e:
self._buffer(buff)
self._exception_handler(e)
break
if len(buff) - prv_len == 0:
self._buffer(buff)
if self._response.chunked and not self._response.fp:
self.progress.force_to_finish_go()
self._callback.put_nowait(SIGNAL_EMPTY_RECV(self))
break
self.progress.go(len(buff) - prv_len)
if self.progress.go_inc >= self.progress.length:
self._buffer(buff)
break
elif len(buff) >= 1048576: # 1 MB
self._buffer(buff)
del buff
buff = b''
self.progress.stop()
def close(self):
response = self._response
connection = self._connection
self._response = None
self._connection = None
self.__connected = False
if response:
response.close()
if connection:
if connection.sock:
connection.sock.shutdown(socket.SHUT_RDWR)
connection.close()
def getheader(self, name, default=None):
return self._response.getheader(name, default)
def run(self, callback):
self._callback = callback
if not self.__signal.empty():
signal = self.__signal.get()
if signal.id == ID_TASK_SLICE:
putrange = self.progress.slice_check(signal.content)
self._callback.put_nowait(
SIGNAL_TASK_SLICE(Slice(client=HTTPClient,
range=putrange)))
elif signal.id == ID_SWITCH:
self.source = signal.content
elif signal.id == ID_TASK_PAUSE:
self._callback = None
callback.put_nowait(SIGNAL_THREAD_END(self))
elif signal.id == ID_WAIT:
sleep(signal.content)
self.__signal.task_done()
try:
if not self.__connected:
self.connect()
self.retrieve()
except (socket.gaierror, socket.timeout, URLUnknownError, HTTP4XXError,
URLError, HTTPError, Exception) as e:
self._exception_handler(e)
finally:
self.close()
self._callback = None
callback.put_nowait(SIGNAL_THREAD_END(self))
def open_only(self, callback):
self._callback = callback
res = None
try:
res = self.connect()
except (socket.gaierror, socket.timeout, URLUnknownError, HTTP4XXError,
URLError, HTTPError, Exception) as e:
self._exception_handler(e)
self._callback = None
return res
def _exception_handler(self, exception):
client_exc = ClientException(client=self, exception=exception)
if type(exception) in (URLError, HTTPError, socket.timeout):
signal_type = SIGNAL_TIMEOUT
elif type(exception) is socket.gaierror:
signal_type = SIGNAL_GAIERROR
elif type(exception) is HTTP4XXError:
signal_type = SIGNAL_CRASH
else:
signal_type = SIGNAL_UNKNOWN
self._callback.put_nowait(SIGNAL_URL_STATUS(signal_type(client_exc)))
def install_callback(self, callback):
self._callback = callback
def clear_callback(self):
self._callback = None
def send_signal(self, signal):
self.__signal.put_nowait(signal)
def write_to_file(self, fp):
if not self.progress.is_empty_buff():
fp.seek(self.progress.begin + self.progress.done_inc)
fp.writelines(self.progress.fetch_buffer())
def getprogress(self):
return self.progress
def getsource(self):
return self.source
def getresonse(self):
return self._response
def clear_signal(self):
del self.__signal
self.__signal = Queue()
