def run(self):
"""Run the proton event/timer loop."""
LOG.debug("Starting Proton thread, container=%s",
self._container.name)
while not self._shutdown:
readfds = [self._requests]
writefds = []
deadline = self._scheduler._next_deadline
pyngus_conn = self._connection and self._connection.pyngus_conn
if pyngus_conn and self._connection.socket:
if pyngus_conn.needs_input:
readfds.append(self._connection)
if pyngus_conn.has_output:
writefds.append(self._connection)
if pyngus_conn.deadline:
deadline = (pyngus_conn.deadline if not deadline else
min(deadline, pyngus_conn.deadline))
# force select to return in time to service the next expiring timer
if deadline:
_now = now()
timeout = 0 if deadline <= _now else (deadline - _now)
else:
timeout = None
# and now we wait...
try:
select.select(readfds, writefds, [], timeout)
except select.error as serror:
if serror[0] == errno.EINTR:
LOG.warning(_LW("ignoring interrupt from select(): %s"),
str(serror))
continue
raise # assuming fatal...
# Ignore the select return value - simply poll the socket for I/O.
# Testing shows that polling improves latency over checking the
# lists returned by select()
self._requests.process_requests()
self._connection.read_socket()
if pyngus_conn and pyngus_conn.deadline:
_now = now()
if pyngus_conn.deadline <= _now:
pyngus_conn.process(_now)
self._connection.write_socket()
self._scheduler._process() # run any deferred requests
LOG.info(_LI("eventloop thread exiting, container=%s"),
self._container.name)
def iter(self, result, exc_info, start_time):
fut = Future(self.statistics['attempt_number'])
if result is not NO_RESULT:
trial_end_time = now()
fut.set_result(result)
retry = self.retry(fut)
elif exc_info:
trial_end_time = now()
t, e, tb = exc_info
_utils.capture(fut, exc_info)
if isinstance(e, TryAgain):
retry = True
else:
retry = self.retry(fut)
else:
if self.before is not None:
self.before(self.fn, self.statistics['attempt_number'])
return DoAttempt()
if not retry:
return fut.result()
if self.after is not None:
trial_time_taken = trial_end_time - start_time
self.after(self.fn, self.statistics['attempt_number'],
trial_time_taken)
delay_since_first_attempt = now() - self.statistics['start_time']
self.statistics['delay_since_first_attempt'] = \
delay_since_first_attempt
if self.stop(self.statistics['attempt_number'],
delay_since_first_attempt):
retry_exc = self.retry_error_cls(fut)
if self.reraise:
raise retry_exc.reraise()
six.raise_from(retry_exc, fut.exception())
if self.wait:
if self._wait_takes_result:
sleep = self.wait(self.statistics['attempt_number'],
delay_since_first_attempt,
last_result=fut)
else:
sleep = self.wait(self.statistics['attempt_number'],
delay_since_first_attempt)
else:
sleep = 0
self.statistics['idle_for'] += sleep
self.statistics['attempt_number'] += 1
return DoSleep(sleep)
def run(self):
"""Run the proton event/timer loop."""
LOG.debug("Starting Proton thread, container=%s", self._container.name)
while not self._shutdown:
readers, writers, timers = self._container.need_processing()
readfds = [c.user_context for c in readers]
# additionally, always check for readability of pipe we
# are using to wakeup processing thread by other threads
readfds.append(self._requests)
writefds = [c.user_context for c in writers]
# force select to return in time to service the next expiring timer
d1 = self._scheduler._next_deadline
d2 = timers[0].deadline if timers else None
deadline = min(d1, d2) if d1 and d2 else d1 if not d2 else d2
if deadline:
_now = now()
timeout = 0 if deadline <= _now else (deadline - _now)
else:
timeout = None
# and now we wait...
try:
results = select.select(readfds, writefds, [], timeout)
except select.error as serror:
if serror[0] == errno.EINTR:
LOG.warning(_LW("ignoring interrupt from select(): %s"),
str(serror))
continue
raise # assuming fatal...
readable, writable, ignore = results
for r in readable:
r.read()
if timers:
_now = now()
for t in timers:
if t.deadline > _now:
break
t.process(_now)
for w in writable:
w.write()
self._scheduler._process() # run any deferred requests
LOG.info(_LI("eventloop thread exiting, container=%s"),
self._container.name)
def run(self):
"""Run the proton event/timer loop."""
LOG.debug("Starting Proton thread, container=%s", self._container.name)
while not self._shutdown:
readfds = [self._requests]
writefds = []
deadline = self._scheduler._next_deadline
pyngus_conn = self._connection and self._connection.pyngus_conn
if pyngus_conn and self._connection.socket:
if pyngus_conn.needs_input:
readfds.append(self._connection)
if pyngus_conn.has_output:
writefds.append(self._connection)
if pyngus_conn.deadline:
deadline = (pyngus_conn.deadline if not deadline else min(
deadline, pyngus_conn.deadline))
# force select to return in time to service the next expiring timer
if deadline:
_now = now()
timeout = 0 if deadline <= _now else (deadline - _now)
else:
timeout = None
# and now we wait...
try:
select.select(readfds, writefds, [], timeout)
except select.error as serror:
if serror[0] == errno.EINTR:
LOG.warning(_LW("ignoring interrupt from select(): %s"),
str(serror))
continue
raise # assuming fatal...
# Ignore the select return value - simply poll the socket for I/O.
# Testing shows that polling improves latency over checking the
# lists returned by select()
self._requests.process_requests()
self._connection.read_socket()
if pyngus_conn and pyngus_conn.deadline:
_now = now()
if pyngus_conn.deadline <= _now:
pyngus_conn.process(_now)
self._connection.write_socket()
self._scheduler._process() # run any deferred requests
LOG.info(_LI("eventloop thread exiting, container=%s"),
self._container.name)
def call(self, fn, *args, **kwargs):
self.begin(fn)
result = NO_RESULT
exc_info = None
start_time = now()
while True:
do = self.iter(result=result,
exc_info=exc_info,
start_time=start_time)
if isinstance(do, DoAttempt):
try:
result = yield from fn(*args, **kwargs)
exc_info = None
continue
except Exception:
result = NO_RESULT
exc_info = sys.exc_info()
continue
elif isinstance(do, DoSleep):
result = NO_RESULT
exc_info = None
yield from self.sleep(do, loop=self.loop)
else:
return do
def run_periodic_tasks(self, context, raise_on_error=False):
"""Tasks to be run at a periodic interval."""
idle_for = DEFAULT_INTERVAL
for task_name, task in self._periodic_tasks:
if (task._periodic_external_ok and not
self.conf.run_external_periodic_tasks):
continue
full_task_name = '.'.join([self.__class__.__name__, task_name])
spacing = self._periodic_spacing[task_name]
last_run = self._periodic_last_run[task_name]
# Check if due, if not skip
idle_for = min(idle_for, spacing)
if last_run is not None:
delta = last_run + spacing - now()
if delta > 0:
idle_for = min(idle_for, delta)
continue
LOG.debug("Running periodic task %(full_task_name)s",
{"full_task_name": full_task_name})
self._periodic_last_run[task_name] = _nearest_boundary(
last_run, spacing)
try:
task(self, context)
except Exception:
if raise_on_error:
raise
LOG.exception(_LE("Error during %(full_task_name)s"),
{"full_task_name": full_task_name})
time.sleep(0)
return idle_for
def run_periodic_tasks(self, context, raise_on_error=False):
"""Tasks to be run at a periodic interval."""
idle_for = DEFAULT_INTERVAL
for task_name, task in self._periodic_tasks:
if (task._periodic_external_ok
and not self.conf.run_external_periodic_tasks):
continue
full_task_name = '.'.join([self.__class__.__name__, task_name])
spacing = self._periodic_spacing[task_name]
last_run = self._periodic_last_run[task_name]
# Check if due, if not skip
idle_for = min(idle_for, spacing)
if last_run is not None:
delta = last_run + spacing - now()
if delta > 0:
idle_for = min(idle_for, delta)
continue
LOG.debug("Running periodic task %(full_task_name)s",
{"full_task_name": full_task_name})
self._periodic_last_run[task_name] = _nearest_boundary(
last_run, spacing)
try:
task(self, context)
except Exception:
if raise_on_error:
raise
LOG.exception(_LE("Error during %(full_task_name)s"),
{"full_task_name": full_task_name})
time.sleep(0)
return idle_for
def begin(self, fn):
self.fn = fn
self.statistics.clear()
self.start_time = now()
self.statistics['start_time'] = self.start_time
self.attempt_number = 1
self.statistics['attempt_number'] = self.attempt_number
self.statistics['idle_for'] = 0
def iter(self, result=NO_RESULT, exc_info=None):
fut = Future(self.attempt_number)
if result is not NO_RESULT:
trial_end_time = now()
fut.set_result(result)
retry = self.retry(fut)
elif exc_info:
trial_end_time = now()
t, e, tb = exc_info
_utils.capture(fut, exc_info)
if isinstance(e, TryAgain):
retry = True
else:
retry = self.retry(fut)
else:
if self.before is not None:
self.before(self.fn, self.attempt_number)
self.trial_start_time = now()
return DoAttempt()
if not retry:
return fut.result()
if self.after is not None:
trial_time_taken = trial_end_time - self.trial_start_time
self.after(self.fn, self.attempt_number, trial_time_taken)
delay_since_first_attempt = now() - self.start_time
self.statistics['delay_since_first_attempt'] = \
delay_since_first_attempt
if self.stop(self.attempt_number, delay_since_first_attempt):
if self.reraise:
raise RetryError(fut).reraise()
six.raise_from(RetryError(fut), fut.exception())
if self.wait:
sleep = self.wait(self.attempt_number, delay_since_first_attempt)
else:
sleep = 0
self.statistics['idle_for'] += sleep
self.attempt_number += 1
self.statistics['attempt_number'] = self.attempt_number
return DoSleep(sleep)
def stop(self):
"""Stops the watch."""
if self._state == self._STOPPED:
return self
if self._state != self._STARTED:
raise RuntimeError("Can not stop a stopwatch that has not been"
" started")
self._stopped_at = now()
self._state = self._STOPPED
return self
def stop(self):
"""Stops the watch."""
if self._state == self._STOPPED:
return self
if self._state != self._STARTED:
raise RuntimeError("Can not stop a stopwatch that has not been"
" started")
self._stopped_at = now()
self._state = self._STOPPED
return self
def write_socket(self):
"""Called to write to the socket."""
if self.socket:
try:
pyngus.write_socket_output(self.pyngus_conn, self.socket)
self.pyngus_conn.process(now())
except (socket.timeout, socket.error) as e:
# pyngus handles EAGAIN/EWOULDBLOCK and EINTER
self.pyngus_conn.close_output()
self.pyngus_conn.close_input()
self._handler.socket_error(str(e))
def write_socket(self):
"""Called to write to the socket."""
if self.socket:
try:
pyngus.write_socket_output(self.pyngus_conn, self.socket)
self.pyngus_conn.process(now())
except (socket.timeout, socket.error) as e:
# pyngus handles EAGAIN/EWOULDBLOCK and EINTER
self.pyngus_conn.close_output()
self.pyngus_conn.close_input()
self._handler.socket_error(str(e))
def start(self):
"""Starts the watch (if not already started).
NOTE(harlowja): resets any splits previously captured (if any).
"""
if self._state == self._STARTED:
return self
self._started_at = now()
self._stopped_at = None
self._state = self._STARTED
self._splits = ()
return self
def elapsed(self, maximum=None):
"""Returns how many seconds have elapsed."""
if self._state not in (self._STARTED, self._STOPPED):
raise RuntimeError("Can not get the elapsed time of a stopwatch"
" if it has not been started/stopped")
if self._state == self._STOPPED:
elapsed = self._delta_seconds(self._started_at, self._stopped_at)
else:
elapsed = self._delta_seconds(self._started_at, now())
if maximum is not None and elapsed > maximum:
elapsed = max(0.0, maximum)
return elapsed
def start(self):
"""Starts the watch (if not already started).
NOTE(harlowja): resets any splits previously captured (if any).
"""
if self._state == self._STARTED:
return self
self._started_at = now()
self._stopped_at = None
self._state = self._STARTED
self._splits = []
return self
def elapsed(self, maximum=None):
"""Returns how many seconds have elapsed."""
if self._state not in (self._STARTED, self._STOPPED):
raise RuntimeError("Can not get the elapsed time of a stopwatch"
" if it has not been started/stopped")
if self._state == self._STOPPED:
elapsed = self._delta_seconds(self._started_at, self._stopped_at)
else:
elapsed = self._delta_seconds(self._started_at, now())
if maximum is not None and elapsed > maximum:
elapsed = max(0.0, maximum)
return elapsed
def _process(self):
"""Invoke all expired callables."""
if self._deadlines:
_now = now()
try:
while self._deadlines[0] <= _now:
deadline = heapq.heappop(self._deadlines)
callbacks = self._callbacks[deadline]
del self._callbacks[deadline]
for cb in callbacks:
cb.callback and cb.callback()
except IndexError:
pass
def _get_delay(self, max_delay=None):
"""Get the delay in milliseconds until the next callable needs to be
run, or 'max_delay' if no outstanding callables or the delay to the
next callable is > 'max_delay'.
"""
due = self._deadlines[0] if self._deadlines else None
if due is None:
return max_delay
_now = now()
if due <= _now:
return 0
else:
return min(due - _now, max_delay) if max_delay else due - _now
def _process(self):
"""Invoke all expired callables."""
if self._deadlines:
_now = now()
try:
while self._deadlines[0] <= _now:
deadline = heapq.heappop(self._deadlines)
callbacks = self._callbacks[deadline]
del self._callbacks[deadline]
for cb in callbacks:
cb.callback and cb.callback()
except IndexError:
pass
def _inspect_cached(self, cache, instance, duration):
cache.setdefault(self.inspector_method, {})
if instance.id not in cache[self.inspector_method]:
result = getattr(self.inspector, self.inspector_method)(instance,
duration)
polled_time = now()
# Ensure we don't cache an iterator
if isinstance(result, collections.Iterable):
result = list(result)
else:
result = [result]
cache[self.inspector_method][instance.id] = (polled_time, result)
return cache[self.inspector_method][instance.id]
def write(self):
"""Called when socket is write-ready."""
while True:
try:
rc = pyngus.write_socket_output(self.connection, self.socket)
self.connection.process(now())
return rc
except (socket.timeout, socket.error) as e:
# pyngus handles EAGAIN/EWOULDBLOCK and EINTER
self.connection.close_output()
self.connection.close_input()
self._handler.socket_error(str(e))
return pyngus.Connection.EOS
def _get_delay(self, max_delay=None):
"""Get the delay in milliseconds until the next callable needs to be
run, or 'max_delay' if no outstanding callables or the delay to the
next callable is > 'max_delay'.
"""
due = self._deadlines[0] if self._deadlines else None
if due is None:
return max_delay
_now = now()
if due <= _now:
return 0
else:
return min(due - _now, max_delay) if max_delay else due - _now
def send(self, send_task):
if send_task.deadline and send_task.deadline <= now():
send_task._on_timeout()
return
if send_task.retry is None or send_task.retry < 0:
send_task.retry = None
key = keyify(send_task.target, send_task.service)
sender = self._senders.get(key)
if not sender:
sender = Sender(send_task.target, self.processor,
self.link_retry_delay, send_task.service)
self._senders[key] = sender
if self.reply_link and self.reply_link.active:
sender.attach(self._socket_connection.connection,
self.reply_link, self.addresser)
sender.send_message(send_task)
def call(self, fn, *args, **kwargs):
self.statistics.clear()
start_time = now()
self.statistics['start_time'] = start_time
attempt_number = 1
self.statistics['attempt_number'] = attempt_number
self.statistics['idle_for'] = 0
while True:
trial_start_time = now()
if self.before is not None:
self.before(fn, attempt_number)
fut = Future(attempt_number)
try:
result = fn(*args, **kwargs)
except TryAgain:
trial_end_time = now()
retry = True
except Exception:
trial_end_time = now()
tb = sys.exc_info()
try:
_utils.capture(fut, tb)
finally:
del tb
retry = self.retry(fut)
else:
trial_end_time = now()
fut.set_result(result)
retry = self.retry(fut)
if not retry:
return fut.result()
if self.after is not None:
trial_time_taken = trial_end_time - trial_start_time
self.after(fn, attempt_number, trial_time_taken)
delay_since_first_attempt = now() - start_time
self.statistics['delay_since_first_attempt'] = \
delay_since_first_attempt
if self.stop(attempt_number, delay_since_first_attempt):
if self.reraise:
raise RetryError(fut).reraise()
six.raise_from(RetryError(fut), fut.exception())
if self.wait:
sleep = self.wait(attempt_number, delay_since_first_attempt)
else:
sleep = 0
self.statistics['idle_for'] += sleep
self.sleep(sleep)
attempt_number += 1
self.statistics['attempt_number'] = attempt_number
def decorator(f):
# Test for old style invocation
if 'ticks_between_runs' in kwargs:
raise InvalidPeriodicTaskArg(arg='ticks_between_runs')
# Control if run at all
f._periodic_task = True
f._periodic_external_ok = kwargs.pop('external_process_ok', False)
f._periodic_enabled = kwargs.pop('enabled', True)
f._periodic_name = kwargs.pop('name', f.__name__)
# Control frequency
f._periodic_spacing = kwargs.pop('spacing', 0)
f._periodic_immediate = kwargs.pop('run_immediately', False)
if f._periodic_immediate:
f._periodic_last_run = None
else:
f._periodic_last_run = now()
return f
def decorator(f):
# Test for old style invocation
if 'ticks_between_runs' in kwargs:
raise InvalidPeriodicTaskArg(arg='ticks_between_runs')
# Control if run at all
f._periodic_task = True
f._periodic_external_ok = kwargs.pop('external_process_ok', False)
f._periodic_enabled = kwargs.pop('enabled', True)
f._periodic_name = kwargs.pop('name', f.__name__)
# Control frequency
f._periodic_spacing = kwargs.pop('spacing', 0)
f._periodic_immediate = kwargs.pop('run_immediately', False)
if f._periodic_immediate:
f._periodic_last_run = None
else:
f._periodic_last_run = now()
return f
def _nearest_boundary(last_run, spacing):
"""Find the nearest boundary in the past.
The boundary is a multiple of the spacing with the last run as an offset.
Eg if last run was 10 and spacing was 7, the new last run could be: 17, 24,
31, 38...
0% to 5% of the spacing value will be added to this value to ensure tasks
do not synchronize. This jitter is rounded to the nearest second, this
means that spacings smaller than 20 seconds will not have jitter.
"""
current_time = now()
if last_run is None:
return current_time
delta = current_time - last_run
offset = delta % spacing
# Add up to 5% jitter
jitter = int(spacing * (random.random() / 20))
return current_time - offset + jitter
def _nearest_boundary(last_run, spacing):
"""Find the nearest boundary in the past.
The boundary is a multiple of the spacing with the last run as an offset.
Eg if last run was 10 and spacing was 7, the new last run could be: 17, 24,
31, 38...
0% to 5% of the spacing value will be added to this value to ensure tasks
do not synchronize. This jitter is rounded to the nearest second, this
means that spacings smaller than 20 seconds will not have jitter.
"""
current_time = now()
if last_run is None:
return current_time
delta = current_time - last_run
offset = delta % spacing
# Add up to 5% jitter
jitter = int(spacing * (random.random() / 20)) # nosec
return current_time - offset + jitter
def run_periodic_tasks(self, context, raise_on_error=False):
"""Tasks to be run at a periodic interval."""
idle_for = DEFAULT_INTERVAL # 60.0
for task_name, task in self._periodic_tasks:
if (task._periodic_external_ok
and not self.conf.run_external_periodic_tasks):
continue
cls_name = reflection.get_class_name(
self, fully_qualified=False) #调用该函数的cls_name，用于log
full_task_name = '.'.join([cls_name, task_name])
spacing = self._periodic_spacing[task_name] #运行时间间隔
last_run = self._periodic_last_run[task_name] #最后运行时间
# Check if due, if not skip
idle_for = min(idle_for, spacing) #最近需要执行的任务，还要多久。
if last_run is not None: #last_run is None则立即run
delta = last_run + spacing - now()
if delta > 0: #时间未到
idle_for = min(idle_for, delta)
continue
LOG.debug("Running periodic task %(full_task_name)s",
{"full_task_name": full_task_name})
self._periodic_last_run[
task_name] = _nearest_boundary( #设定最后运行时间加入了少许偏移量。
last_run, spacing)
try:
task(self, context)
except BaseException:
if raise_on_error:
raise
LOG.exception("Error during %(full_task_name)s",
{"full_task_name": full_task_name})
time.sleep(0)
return idle_for
def elapsed(self):
return max(0.0, now() - self.started_at)
def compute_timeout(offset):
# minimize the timer granularity to one second so we don't have to track
# too many timers
return math.ceil(now() + offset)
def compute_timeout(offset):
# minimize the timer granularity to one second so we don't have to track
# too many timers
return math.ceil(now() + offset)
def __exit__(self, exc_type, exc_value, exc_tb):
self.stopped_at = now()
def elapsed(self):
if self.stopped_at is not None:
end_time = self.stopped_at
else:
end_time = now()
return max(0.0, end_time - self.started_at)
def __init__(self):
self.started_at = now()
def start(self):
self.started_at = now()
self.stopped_at = None
def elapsed(self):
if self.stopped_at is not None:
end_time = self.stopped_at
else:
end_time = now()
return max(0.0, end_time - self.started_at)
def __exit__(self, exc_type, exc_value, exc_tb):
self.stopped_at = now()
def start(self):
self.started_at = now()
self.stopped_at = None