def _run_transport(self, maximumDuration=None, txonly=False,
incomingHandler=None):
# This is where multiple external threads are synchronized for
# receives. Transmits will flow down into the transmit layer
# where they are queued with thread safety, but threads
# blocking on a receive will all be lined up through this point.
max_runtime = ExpirationTimer(maximumDuration)
with self._cv:
while self._transport_runner:
self._cv.wait(max_runtime.remainingSeconds())
if max_runtime.expired():
return None
self._transport_runner = True
try:
r = Thespian__UpdateWork()
while isinstance(r, Thespian__UpdateWork):
r = self.transport.run(TransmitOnly if txonly else incomingHandler,
max_runtime.remaining())
return r
# incomingHandler callback could deadlock on this same thread; is it ever not None?
finally:
with self._cv:
self._transport_runner = False
self._cv.notify()
def ask(self, anActor, msg, timeout):
txwatch = self._tx_to_actor(anActor, msg) # KWQ: pass timeout on tx??
askLimit = ExpirationTimer(toTimeDeltaOrNone(timeout))
while not askLimit.expired():
response = self._run_transport(askLimit.remaining())
if txwatch.failed:
if txwatch.failure in [SendStatus.DeadTarget,
SendStatus.Failed,
SendStatus.NotSent]:
# Silent failure; not all transports can indicate
# this, so for conformity the Dead Letter handler is
# the intended method of handling this issue.
return None
raise ActorSystemFailure('Transmit of ask message to %s failed (%s)'%(
str(anActor),
str(txwatch.failure)))
if not isinstance(response, ReceiveEnvelope):
# Timed out or other failure, give up.
break
# Do not send miscellaneous ActorSystemMessages to the
# caller that it might not recognize. If one of those was
# recieved, loop to get another response.
if not isInternalActorSystemMessage(response.message):
return response.message
return None
def tell(self, anActor, msg):
attemptLimit = ExpirationTimer(MAX_TELL_PERIOD)
# transport may not use sockets, but this helps error handling
# in case it does.
import socket
for attempt in range(5000):
try:
txwatch = self._tx_to_actor(anActor, msg)
while not attemptLimit.expired():
if not self._run_transport(attemptLimit.remaining(),
txonly=True):
# all transmits completed
return
if txwatch.failed:
raise ActorSystemFailure(
'Error sending to %s: %s' % (str(anActor),
str(txwatch.failure)))
raise ActorSystemRequestTimeout(
'Unable to send to %s within %s' %
(str(anActor), str(MAX_TELL_PERIOD)))
except socket.error as ex:
import errno
if errno.EMFILE == ex.errno:
import time
time.sleep(0.1)
else:
raise
def _runSends(self, timeout=None, stop_on_available=False):
numsends = 0
endtime = ExpirationTimer(toTimeDeltaOrNone(timeout))
while not endtime.expired():
while self._pendingSends:
numsends += 1
if self.procLimit and numsends > self.procLimit:
raise RuntimeError('Too many sends')
self._realizeWakeups()
with self._private_lock:
try:
nextmsg = self._pendingSends.pop(0)
except IndexError:
pass
else:
self._runSingleSend(nextmsg)
if stop_on_available and \
any([not isInternalActorSystemMessage(M)
for M in getattr(stop_on_available.instance,
'responses', [])]):
return
if endtime.remaining(forever=-1) == -1:
return
next_wakeup = self._next_wakeup()
if next_wakeup is None or next_wakeup > endtime:
return
time.sleep(max(0, timePeriodSeconds(next_wakeup.remaining())))
self._realizeWakeups()
def shutdown(self):
thesplog('ActorSystem shutdown requested.', level=logging.INFO)
time_to_quit = ExpirationTimer(MAX_SYSTEM_SHUTDOWN_DELAY)
txwatch = self._tx_to_admin(SystemShutdown())
while not time_to_quit.expired():
response = self._run_transport(time_to_quit.remaining())
if txwatch.failed:
thesplog('Could not send shutdown request to Admin'
'; aborting but not necessarily stopped',
level=logging.WARNING)
return
if isinstance(response, ReceiveEnvelope):
if isinstance(response.message, SystemShutdownCompleted):
break
else:
thesplog('Expected shutdown completed message, got: %s', response.message,
level=logging.WARNING)
elif isinstance(response, (Thespian__Run_Expired,
Thespian__Run_Terminated,
Thespian__Run_Expired)):
break
else:
thesplog('No response to Admin shutdown request; Actor system not completely shutdown',
level=logging.ERROR)
self.transport.close()
thesplog('ActorSystem shutdown complete.')
def shutdown(self):
thesplog('ActorSystem shutdown requested.', level=logging.INFO)
time_to_quit = ExpirationTimer(MAX_SYSTEM_SHUTDOWN_DELAY)
txwatch = self._tx_to_admin(SystemShutdown())
while not time_to_quit.expired():
response = self._run_transport(time_to_quit.remaining())
if txwatch.failed:
thesplog(
'Could not send shutdown request to Admin'
'; aborting but not necessarily stopped',
level=logging.WARNING)
return
if isinstance(response, ReceiveEnvelope):
if isinstance(response.message, SystemShutdownCompleted):
break
else:
thesplog('Expected shutdown completed message, got: %s',
response.message,
level=logging.WARNING)
elif isinstance(response,
(Thespian__Run_Expired, Thespian__Run_Terminated,
Thespian__Run_Expired)):
break
else:
thesplog(
'No response to Admin shutdown request; Actor system not completely shutdown',
level=logging.ERROR)
self.transport.close()
thesplog('ActorSystem shutdown complete.')
def _drain_tx_queue_if_needed(self, max_delay=None):
v, _ = self._complete_expired_intents()
if v >= MAX_QUEUED_TRANSMITS and self._aTB_rx_pause_enabled:
# Try to drain our local work before accepting more
# because it looks like we're getting really behind. This
# is dangerous though, because if other Actors are having
# the same issue this can create a deadlock.
thesplog(
'Entering tx-only mode to drain excessive queue'
' (%s > %s, drain-to %s)',
v,
MAX_QUEUED_TRANSMITS,
QUEUE_TRANSMIT_UNBLOCK_THRESHOLD,
level=logging.WARNING)
finish_time = ExpirationTimer(max_delay if max_delay else None)
while v > QUEUE_TRANSMIT_UNBLOCK_THRESHOLD and not finish_time.expired(
):
if 0 == self.run(TransmitOnly, finish_time.remaining()):
thesplog(
'Exiting tx-only mode because no transport work available.'
)
break
v, _ = self._complete_expired_intents()
thesplog('Exited tx-only mode after draining excessive queue (%s)',
len(self._aTB_queuedPendingTransmits),
level=logging.WARNING)
def ask(self, anActor, msg, timeout):
txwatch = self._tx_to_actor(anActor, msg) # KWQ: pass timeout on tx??
askLimit = ExpirationTimer(toTimeDeltaOrNone(timeout))
while not askLimit.expired():
response = self._run_transport(askLimit.remaining())
if txwatch.failed:
if txwatch.failure in [
SendStatus.DeadTarget, SendStatus.Failed,
SendStatus.NotSent
]:
# Silent failure; not all transports can indicate
# this, so for conformity the Dead Letter handler is
# the intended method of handling this issue.
return None
raise ActorSystemFailure(
'Transmit of ask message to %s failed (%s)' %
(str(anActor), str(txwatch.failure)))
if not isinstance(response, ReceiveEnvelope):
# Timed out or other failure, give up.
break
# Do not send miscellaneous ActorSystemMessages to the
# caller that it might not recognize. If one of those was
# recieved, loop to get another response.
if not isInternalActorSystemMessage(response.message):
return response.message
return None
def tell(self, anActor, msg):
attemptLimit = ExpirationTimer(MAX_TELL_PERIOD)
# transport may not use sockets, but this helps error handling
# in case it does.
import socket
for attempt in range(5000):
try:
txwatch = self._tx_to_actor(anActor, msg)
while not attemptLimit.expired():
if not self._run_transport(attemptLimit.remaining(),
txonly=True):
# all transmits completed
return
if txwatch.failed:
raise ActorSystemFailure(
'Error sending to %s: %s' %
(str(anActor), str(txwatch.failure)))
raise ActorSystemRequestTimeout(
'Unable to send to %s within %s' %
(str(anActor), str(MAX_TELL_PERIOD)))
except socket.error as ex:
import errno
if errno.EMFILE == ex.errno:
import time
time.sleep(0.1)
else:
raise
def _run_transport(self, maximumDuration=None, txonly=False,
incomingHandler=None):
# This is where multiple external threads are synchronized for
# receives. Transmits will flow down into the transmit layer
# where they are queued with thread safety, but threads
# blocking on a receive will all be lined up through this point.
max_runtime = ExpirationTimer(maximumDuration)
with self._cv:
while self._transport_runner:
self._cv.wait(max_runtime.remainingSeconds())
if max_runtime.expired():
return None
self._transport_runner = True
try:
r = Thespian__UpdateWork()
while isinstance(r, Thespian__UpdateWork):
r = self.transport.run(TransmitOnly if txonly else incomingHandler,
max_runtime.remaining())
return r
# incomingHandler callback could deadlock on this same thread; is it ever not None?
finally:
with self._cv:
self._transport_runner = False
self._cv.notify()
class PauseWithBackoff(object):
def backoffPause(self, startPausing=False):
if startPausing:
self._lastPauseLength = backoffDelay(getattr(self, '_lastPauseLength', 0))
self._pauseUntil = ExpirationTimer(self._lastPauseLength)
return self._lastPauseLength
elif hasattr(self, '_pauseUntil'):
if not self._pauseUntil.expired():
return self._pauseUntil.remaining()
delattr(self, '_pauseUntil')
return timedelta(0)
class PauseWithBackoff(object):
def backoffPause(self, startPausing=False):
if startPausing:
self._lastPauseLength = backoffDelay(
getattr(self, '_lastPauseLength', 0))
self._pauseUntil = ExpirationTimer(self._lastPauseLength)
return self._lastPauseLength
elif hasattr(self, '_pauseUntil'):
if not self._pauseUntil.expired():
return self._pauseUntil.remaining()
delattr(self, '_pauseUntil')
return timedelta(0)
def unloadActorSource(self, sourceHash):
loadLimit = ExpirationTimer(MAX_LOAD_SOURCE_DELAY)
txwatch = self._tx_to_admin(ValidateSource(sourceHash, None))
while not loadLimit.expired():
if not self._run_transport(loadLimit.remaining(), txonly=True):
return # all transmits completed
if txwatch.failed:
raise ActorSystemFailure(
'Error sending source unload to Admin: %s' %
str(txwatch.failure))
raise ActorSystemRequestTimeout('Unload source timeout: ' +
str(loadLimit))
def unloadActorSource(self, sourceHash):
loadLimit = ExpirationTimer(MAX_LOAD_SOURCE_DELAY)
txwatch = self._tx_to_admin(ValidateSource(sourceHash, None))
while not loadLimit.expired():
if not self._run_transport(loadLimit.remaining(), txonly=True):
return # all transmits completed
if txwatch.failed:
raise ActorSystemFailure(
'Error sending source unload to Admin: %s' %
str(txwatch.failure))
raise ActorSystemRequestTimeout('Unload source timeout: ' +
str(loadLimit))
def updateCapability(self, capabilityName, capabilityValue=None):
attemptLimit = ExpirationTimer(MAX_CAPABILITY_UPDATE_DELAY)
txwatch = self._tx_to_admin(
CapabilityUpdate(capabilityName, capabilityValue))
while not attemptLimit.expired():
if not self._run_transport(attemptLimit.remaining(), txonly=True):
return # all transmits completed
if txwatch.failed:
raise ActorSystemFailure(
'Error sending capability updates to Admin: %s' %
str(txwatch.failure))
raise ActorSystemRequestTimeout(
'Unable to confirm capability update in %s' %
str(MAX_CAPABILITY_UPDATE_DELAY))
def updateCapability(self, capabilityName, capabilityValue=None):
attemptLimit = ExpirationTimer(MAX_CAPABILITY_UPDATE_DELAY)
txwatch = self._tx_to_admin(CapabilityUpdate(capabilityName,
capabilityValue))
while not attemptLimit.expired():
if not self._run_transport(attemptLimit.remaining(), txonly=True):
return # all transmits completed
if txwatch.failed:
raise ActorSystemFailure(
'Error sending capability updates to Admin: %s' %
str(txwatch.failure))
raise ActorSystemRequestTimeout(
'Unable to confirm capability update in %s' %
str(MAX_CAPABILITY_UPDATE_DELAY))
def _drain_tx_queue_if_needed(self, max_delay=None):
v, _ = self._complete_expired_intents()
if v >= MAX_QUEUED_TRANSMITS and self._aTB_rx_pause_enabled:
# Try to drain our local work before accepting more
# because it looks like we're getting really behind. This
# is dangerous though, because if other Actors are having
# the same issue this can create a deadlock.
thesplog('Entering tx-only mode to drain excessive queue'
' (%s > %s, drain-to %s)',
v, MAX_QUEUED_TRANSMITS,
QUEUE_TRANSMIT_UNBLOCK_THRESHOLD,
level=logging.WARNING)
finish_time = ExpirationTimer(max_delay if max_delay else None)
while v > QUEUE_TRANSMIT_UNBLOCK_THRESHOLD and not finish_time.expired():
if 0 == self.run(TransmitOnly, finish_time.remaining()):
thesplog('Exiting tx-only mode because no transport work available.')
break
v, _ = self._complete_expired_intents()
thesplog('Exited tx-only mode after draining excessive queue (%s)',
len(self._aTB_queuedPendingTransmits),
level=logging.WARNING)
def loadActorSource(self, fname):
loadLimit = ExpirationTimer(MAX_LOAD_SOURCE_DELAY)
f = fname if hasattr(fname, 'read') else open(fname, 'rb')
try:
d = f.read()
import hashlib
hval = hashlib.md5(d).hexdigest()
txwatch = self._tx_to_admin(
ValidateSource(
hval, d,
getattr(f, 'name',
str(fname) if hasattr(fname, 'read') else fname)))
while not loadLimit.expired():
if not self._run_transport(loadLimit.remaining(), txonly=True):
# All transmits completed
return hval
if txwatch.failed:
raise ActorSystemFailure(
'Error sending source load to Admin: %s' %
str(txwatch.failure))
raise ActorSystemRequestTimeout('Load source timeout: ' +
str(loadLimit))
finally:
f.close()
def loadActorSource(self, fname):
loadLimit = ExpirationTimer(MAX_LOAD_SOURCE_DELAY)
f = fname if hasattr(fname, 'read') else open(fname, 'rb')
try:
d = f.read()
import hashlib
hval = hashlib.md5(d).hexdigest()
txwatch = self._tx_to_admin(
ValidateSource(hval, d, getattr(f, 'name',
str(fname)
if hasattr(fname, 'read')
else fname)))
while not loadLimit.expired():
if not self._run_transport(loadLimit.remaining(), txonly=True):
# All transmits completed
return hval
if txwatch.failed:
raise ActorSystemFailure(
'Error sending source load to Admin: %s' %
str(txwatch.failure))
raise ActorSystemRequestTimeout('Load source timeout: ' +
str(loadLimit))
finally:
f.close()
class TransmitIntent(PauseWithBackoff):
"""An individual transmission of data can be encapsulated by a
"transmit intent", which identifies the message and the target
address, and which has a callback for eventual success or
failure indication. Transmit intents may be chained together
to represent a series of outbound transmits. Adding a transmit
intent to the chain may block when the chain reaches an upper
threshold, and remain blocked until enough transmits have
occured (successful or failed) to reduce the size of the chain
below a minimum threshold. This acts to implement server-side
flow control in the system as a whole (although it can
introduce a deadlock scenario if multiple actors form a
transmit loop that is blocked at any point in the loop, so a
transmit intent will fail if it reaches a maximum number of
retries without success).
The TransmitIntent is constructed with a target address, the
message to send, and optional onSuccess and onError callbacks
(both defaulting to None). The callbacks are passed the
TransmitIntent when the transport is finished with it,
selecting the appropriate callback based on the completion
status (the `result' property will reveal the SendStatus actual
result of the attempt). A callback of None will simply discard
the TransmitIntent without passing it to a callback.
The TransmitIntent is passed to the transport that should
perform the intent; the transport may attach its own additional
data to the intent during that processing.
"""
def __init__(self, targetAddr, msg, onSuccess=None, onError=None, maxPeriod=None,
retryPeriod=TRANSMIT_RETRY_PERIOD):
super(TransmitIntent, self).__init__()
self._targetAddr = targetAddr
self._message = msg
self._callbackTo = ResultCallback(onSuccess, onError)
self._resultsts = None
self._quitTime = ExpirationTimer(maxPeriod or DEFAULT_MAX_TRANSMIT_PERIOD)
self._attempts = 0
self.transmit_retry_period = retryPeriod
@property
def targetAddr(self): return self._targetAddr
@property
def message(self): return self._message
def changeTargetAddr(self, newAddr): self._targetAddr = newAddr
def changeMessage(self, newMessage): self._message = newMessage
@property
def result(self): return self._resultsts
@result.setter
def result(self, setResult):
if not isinstance(setResult, SendStatus.BASE):
raise TypeError('TransmitIntent result must be a SendStatus (got %s)'%type(setResult))
self._resultsts = setResult
def completionCallback(self):
"This is called by the transport to perform the success or failure callback operation."
if not self.result:
if self.result == SendStatus.DeadTarget:
# Do not perform logging in case admin or logdirector
# is dead (this will recurse infinitely).
# logging.getLogger('Thespian').warning('Dead target: %s', self.targetAddr)
pass
else:
thesplog('completion error: %s', str(self), level=logging.INFO)
self._callbackTo.resultCallback(self.result, self)
def addCallback(self, onSuccess=None, onFailure=None):
self._callbackTo = ResultCallback(onSuccess, onFailure, self._callbackTo)
def tx_done(self, status):
self.result = status
self.completionCallback()
def awaitingTXSlot(self):
self._awaitingTXSlot = True
def retry(self, immediately=False):
if self._attempts > MAX_TRANSMIT_RETRIES:
return False
if self._quitTime.expired():
return False
self._attempts += 1
if immediately:
self._retryTime = ExpirationTimer(0)
else:
self._retryTime = ExpirationTimer(self._attempts * self.transmit_retry_period)
return True
def timeToRetry(self, socketAvail=False):
if socketAvail and hasattr(self, '_awaitingTXSlot'):
delattr(self, '_awaitingTXSlot')
if hasattr(self, '_retryTime'):
retryNow = self._retryTime.expired()
if retryNow:
delattr(self, '_retryTime')
return retryNow
return socketAvail
def delay(self):
if getattr(self, '_awaitingTXSlot', False):
if self._quitTime.expired():
return timedelta(seconds=0)
return max(timedelta(milliseconds=10), (self._quitTime.remaining()) / 2)
return max(timedelta(seconds=0),
min(self._quitTime.remaining(),
getattr(self, '_retryTime', self._quitTime).remaining(),
getattr(self, '_pauseUntil', self._quitTime).remaining()))
def expired(self):
return self._quitTime.expired()
def __str__(self):
return '************* %s' % self.identify()
def identify(self):
try:
smsg = str(self.message)
except Exception:
smsg = '<msg-cannot-convert-to-ascii>'
if len(smsg) > MAX_SHOWLEN:
smsg = smsg[:MAX_SHOWLEN] + '...'
return 'TransportIntent(' + '-'.join(filter(None, [
str(self.targetAddr),
'pending' if self.result is None else '='+str(self.result),
'' if self.result is not None else 'ExpiresIn_' + str(self.delay()),
'WAITSLOT' if getattr(self, '_awaitingTXSlot', False) else None,
'retry#%d'%self._attempts if self._attempts else '',
str(type(self.message)), smsg,
'quit_%s'%str(self._quitTime.remaining()),
'retry_%s'%str(self._retryTime.remaining()) if getattr(self, '_retryTime', None) else None,
'pause_%s'%str(self._pauseUntil.remaining()) if getattr(self, '_pauseUntil', None) else None,
])) + ')'
class TransmitIntent(PauseWithBackoff):
"""An individual transmission of data can be encapsulated by a
"transmit intent", which identifies the message and the target
address, and which has a callback for eventual success or
failure indication. Transmit intents may be chained together
to represent a series of outbound transmits. Adding a transmit
intent to the chain may block when the chain reaches an upper
threshold, and remain blocked until enough transmits have
occured (successful or failed) to reduce the size of the chain
below a minimum threshold. This acts to implement server-side
flow control in the system as a whole (although it can
introduce a deadlock scenario if multiple actors form a
transmit loop that is blocked at any point in the loop, so a
transmit intent will fail if it reaches a maximum number of
retries without success).
The TransmitIntent is constructed with a target address, the
message to send, and optional onSuccess and onError callbacks
(both defaulting to None). The callbacks are passed the
TransmitIntent when the transport is finished with it,
selecting the appropriate callback based on the completion
status (the `result' property will reveal the SendStatus actual
result of the attempt). A callback of None will simply discard
the TransmitIntent without passing it to a callback.
The TransmitIntent is passed to the transport that should
perform the intent; the transport may attach its own additional
data to the intent during that processing.
"""
def __init__(self,
targetAddr,
msg,
onSuccess=None,
onError=None,
maxPeriod=None,
retryPeriod=TRANSMIT_RETRY_PERIOD):
super(TransmitIntent, self).__init__()
self._targetAddr = targetAddr
self._message = msg
self._callbackTo = ResultCallback(onSuccess, onError)
self._resultsts = None
self._quitTime = ExpirationTimer(maxPeriod
or DEFAULT_MAX_TRANSMIT_PERIOD)
self._attempts = 0
self.transmit_retry_period = retryPeriod
@property
def targetAddr(self):
return self._targetAddr
@property
def message(self):
return self._message
def changeTargetAddr(self, newAddr):
self._targetAddr = newAddr
def changeMessage(self, newMessage):
self._message = newMessage
@property
def result(self):
return self._resultsts
@result.setter
def result(self, setResult):
if not isinstance(setResult, SendStatus.BASE):
raise TypeError(
'TransmitIntent result must be a SendStatus (got %s)' %
type(setResult))
self._resultsts = setResult
def completionCallback(self):
"This is called by the transport to perform the success or failure callback operation."
if not self.result:
if self.result == SendStatus.DeadTarget:
# Do not perform logging in case admin or logdirector
# is dead (this will recurse infinitely).
# logging.getLogger('Thespian').warning('Dead target: %s', self.targetAddr)
pass
else:
thesplog('completion error: %s', str(self), level=logging.INFO)
self._callbackTo.resultCallback(self.result, self)
def addCallback(self, onSuccess=None, onFailure=None):
self._callbackTo = ResultCallback(onSuccess, onFailure,
self._callbackTo)
def tx_done(self, status):
self.result = status
self.completionCallback()
def awaitingTXSlot(self):
self._awaitingTXSlot = True
def retry(self, immediately=False):
if self._attempts > MAX_TRANSMIT_RETRIES:
return False
if self._quitTime.expired():
return False
self._attempts += 1
if immediately:
self._retryTime = ExpirationTimer(0)
else:
self._retryTime = ExpirationTimer(self._attempts *
self.transmit_retry_period)
return True
def timeToRetry(self, socketAvail=False):
if socketAvail and hasattr(self, '_awaitingTXSlot'):
delattr(self, '_awaitingTXSlot')
if hasattr(self, '_retryTime'):
retryNow = self._retryTime.expired()
if retryNow:
delattr(self, '_retryTime')
return retryNow
return socketAvail
def delay(self):
if getattr(self, '_awaitingTXSlot', False):
if self._quitTime.expired():
return timedelta(seconds=0)
return max(timedelta(milliseconds=10),
(self._quitTime.remaining()) / 2)
return max(
timedelta(seconds=0),
min(self._quitTime.remaining(),
getattr(self, '_retryTime', self._quitTime).remaining(),
getattr(self, '_pauseUntil', self._quitTime).remaining()))
def expired(self):
return self._quitTime.expired()
def __str__(self):
return '************* %s' % self.identify()
def identify(self):
try:
smsg = str(self.message)
except Exception:
smsg = '<msg-cannot-convert-to-ascii>'
if len(smsg) > MAX_SHOWLEN:
smsg = smsg[:MAX_SHOWLEN] + '...'
return 'TransportIntent(' + '-'.join(
filter(None, [
str(self.targetAddr),
'pending' if self.result is None else '=' + str(self.result),
'' if self.result is not None else 'ExpiresIn_' +
str(self.delay()),
'WAITSLOT'
if getattr(self, '_awaitingTXSlot', False) else None,
'retry#%d' % self._attempts if self._attempts else '',
str(type(self.message)),
smsg,
'quit_%s' % str(self._quitTime.remaining()),
'retry_%s' % str(self._retryTime.remaining()) if getattr(
self, '_retryTime', None) else None,
'pause_%s' % str(self._pauseUntil.remaining()) if getattr(
self, '_pauseUntil', None) else None,
])) + ')'
def testZeroRemaining(self):
et = ExpirationTimer(timedelta(seconds=0))
assert timedelta(days=0) == et.remaining()
def testNoneRemaining(self):
et = ExpirationTimer(None)
assert et.remaining() is None
def testNoneRemainingExplicitForever(self):
et = ExpirationTimer(None)
assert 5 == et.remaining(5)
def testNonZeroRemaining(self):
et = ExpirationTimer(timedelta(milliseconds=10))
assert timedelta(days=0) < et.remaining()
assert timedelta(milliseconds=11) > et.remaining()
sleep(et.remainingSeconds())
assert timedelta(days=0) == et.remaining()
def testNoneRemainingExplicitForever(self):
et = ExpirationTimer(None)
assert 5 == et.remaining(5)
def testNonZeroRemaining(self):
et = ExpirationTimer(timedelta(milliseconds=10))
assert timedelta(days=0) < et.remaining()
assert timedelta(milliseconds=11) > et.remaining()
sleep(et.remainingSeconds())
assert timedelta(days=0) == et.remaining()
def testZeroRemaining(self):
et = ExpirationTimer(timedelta(seconds=0))
assert timedelta(days=0) == et.remaining()
def testNoneRemaining(self):
et = ExpirationTimer(None)
assert et.remaining() is None
class HysteresisDelaySender(object):
"""Implements hysteresis delay for sending messages. This is intended
to be used for messages exchanged between convention members to
ensure that a mis-behaved member doesn't have the ability to
inflict damage on the entire convention. The first time a
message is sent via this sender it is passed on through, but
that starts a blackout period that starts with the
CONVENTION_HYSTERESIS_MIN_PERIOD. Each additional send attempt
during that blackout period will cause the blackout period to
be extended by the CONVENTION_HYSTERESIS_RATE, up to the
CONVENTION_HYSTERESIS_MAX_PERIOD. Once the blackout period
ends, the queued sends will be sent, but only the last
attempted message of each type for the specified remote target.
At that point, the hysteresis delay will be reduced by the
CONVENTION_HYSTERESIS_RATE; further send attempts will affect
the hysteresis blackout period as described as above but lack
of sending attempts will continue to reduce the hysteresis back
to a zero-delay setting.
Note: delays are updated in a target-independent manner; the
target is only considered when eliminating duplicates.
Note: maxDelay on TransmitIntents is ignored by hysteresis
delays. It is assumed that a transmit intent's maxDelay
is greater than the maximum hysteresis period and/or that
the hysteresis delay is more important than the transmit
intent timeout.
"""
def __init__(self, actual_sender,
hysteresis_min_period = HYSTERESIS_MIN_PERIOD,
hysteresis_max_period = HYSTERESIS_MAX_PERIOD,
hysteresis_rate = HYSTERESIS_RATE):
self._sender = actual_sender
self._hysteresis_until = ExpirationTimer(timedelta(seconds=0))
self._hysteresis_queue = []
self._current_hysteresis = None # timedelta
self._hysteresis_min_period = hysteresis_min_period
self._hysteresis_max_period = hysteresis_max_period
self._hysteresis_rate = hysteresis_rate
@property
def delay(self):
return self._hysteresis_until
def _has_hysteresis(self):
return (self._current_hysteresis is not None and
self._current_hysteresis >= self._hysteresis_min_period)
def _increase_hysteresis(self):
if self._has_hysteresis():
try:
self._current_hysteresis = min(
(self._current_hysteresis * self._hysteresis_rate),
self._hysteresis_max_period)
except TypeError:
# See note below for _decrease_hysteresis
self._current_hysteresis = min(
timedelta(
seconds=(self._current_hysteresis.seconds *
self._hysteresis_rate)),
self._hysteresis_max_period)
else:
self._current_hysteresis = self._hysteresis_min_period
def _decrease_hysteresis(self):
try:
self._current_hysteresis = (
(self._current_hysteresis / self._hysteresis_rate)
if self._has_hysteresis() else None)
except TypeError:
# Python 2.x cannot multiply or divide a timedelta by a
# fractional amount. There is also not a total_seconds
# retrieval from a timedelta, but it should be safe to
# assume that the hysteresis value is not greater than 1
# day.
self._current_hysteresis = timedelta(
seconds=(self._current_hysteresis.seconds /
self._hysteresis_rate)) \
if self._has_hysteresis() else None
def _update_remaining_hysteresis_period(self, reset=False):
if not self._current_hysteresis:
self._hysteresis_until = ExpirationTimer(timedelta(seconds=0))
else:
if reset or not self._hysteresis_until:
self._hysteresis_until = ExpirationTimer(self._current_hysteresis)
else:
self._hysteresis_until = ExpirationTimer(
self._current_hysteresis -
self._hysteresis_until.remaining())
def checkSends(self):
if self.delay.expired():
self._decrease_hysteresis()
self._update_remaining_hysteresis_period(reset=True)
for intent in self._keepIf(lambda M: False):
self._sender(intent)
@staticmethod
def safe_cmp(val1, val2):
try:
return val1 == val2
except Exception:
return False
def sendWithHysteresis(self, intent):
if self._hysteresis_until.expired():
self._current_hysteresis = self._hysteresis_min_period
self._sender(intent)
else:
dups = self._keepIf(lambda M:
(M.targetAddr != intent.targetAddr or
not HysteresisDelaySender
.safe_cmp(M.message, intent.message)))
# The dups are duplicate sends to the new intent's target;
# complete them when the actual message is finally sent
# with the same result
if dups:
intent.addCallback(self._dupSentGood(dups),
self._dupSentFail(dups))
self._hysteresis_queue.append(intent)
self._increase_hysteresis()
self._update_remaining_hysteresis_period()
def cancelSends(self, remoteAddr):
for each in self._keepIf(lambda M: M.targetAddr != remoteAddr):
each.tx_done(SendStatus.Failed)
def _keepIf(self, keepFunc):
requeues, removes = partition(keepFunc, self._hysteresis_queue)
self._hysteresis_queue = requeues
return removes
@staticmethod
def _dupSentGood(dups):
def _finishDups(result, finishedIntent):
for each in dups:
each.tx_done(result)
return _finishDups
@staticmethod
def _dupSentFail(dups):
def _finishDups(result, finishedIntent):
for each in dups:
each.tx_done(result)
return _finishDups
class HysteresisDelaySender(object):
"""Implements hysteresis delay for sending messages. This is intended
to be used for messages exchanged between convention members to
ensure that a mis-behaved member doesn't have the ability to
inflict damage on the entire convention. The first time a
message is sent via this sender it is passed on through, but
that starts a blackout period that starts with the
CONVENTION_HYSTERESIS_MIN_PERIOD. Each additional send attempt
during that blackout period will cause the blackout period to
be extended by the CONVENTION_HYSTERESIS_RATE, up to the
CONVENTION_HYSTERESIS_MAX_PERIOD. Once the blackout period
ends, the queued sends will be sent, but only the last
attempted message of each type for the specified remote target.
At that point, the hysteresis delay will be reduced by the
CONVENTION_HYSTERESIS_RATE; further send attempts will affect
the hysteresis blackout period as described as above but lack
of sending attempts will continue to reduce the hysteresis back
to a zero-delay setting.
Note: delays are updated in a target-independent manner; the
target is only considered when eliminating duplicates.
Note: maxDelay on TransmitIntents is ignored by hysteresis
delays. It is assumed that a transmit intent's maxDelay
is greater than the maximum hysteresis period and/or that
the hysteresis delay is more important than the transmit
intent timeout.
"""
def __init__(self,
actual_sender,
hysteresis_min_period=HYSTERESIS_MIN_PERIOD,
hysteresis_max_period=HYSTERESIS_MAX_PERIOD,
hysteresis_rate=HYSTERESIS_RATE):
self._sender = actual_sender
self._hysteresis_until = ExpirationTimer(timedelta(seconds=0))
self._hysteresis_queue = []
self._current_hysteresis = None # timedelta
self._hysteresis_min_period = hysteresis_min_period
self._hysteresis_max_period = hysteresis_max_period
self._hysteresis_rate = hysteresis_rate
@property
def delay(self):
return self._hysteresis_until
def _has_hysteresis(self):
return (self._current_hysteresis is not None
and self._current_hysteresis >= self._hysteresis_min_period)
def _increase_hysteresis(self):
if self._has_hysteresis():
try:
self._current_hysteresis = min(
(self._current_hysteresis * self._hysteresis_rate),
self._hysteresis_max_period)
except TypeError:
# See note below for _decrease_hysteresis
self._current_hysteresis = min(
timedelta(seconds=(self._current_hysteresis.seconds *
self._hysteresis_rate)),
self._hysteresis_max_period)
else:
self._current_hysteresis = self._hysteresis_min_period
def _decrease_hysteresis(self):
try:
self._current_hysteresis = ((self._current_hysteresis /
self._hysteresis_rate)
if self._has_hysteresis() else None)
except TypeError:
# Python 2.x cannot multiply or divide a timedelta by a
# fractional amount. There is also not a total_seconds
# retrieval from a timedelta, but it should be safe to
# assume that the hysteresis value is not greater than 1
# day.
self._current_hysteresis = timedelta(
seconds=(self._current_hysteresis.seconds /
self._hysteresis_rate)) \
if self._has_hysteresis() else None
def _update_remaining_hysteresis_period(self, reset=False):
if not self._current_hysteresis:
self._hysteresis_until = ExpirationTimer(timedelta(seconds=0))
else:
if reset or not self._hysteresis_until:
self._hysteresis_until = ExpirationTimer(
self._current_hysteresis)
else:
self._hysteresis_until = ExpirationTimer(
self._current_hysteresis -
self._hysteresis_until.remaining())
def checkSends(self):
if self.delay.expired():
self._decrease_hysteresis()
self._update_remaining_hysteresis_period(reset=True)
for intent in self._keepIf(lambda M: False):
self._sender(intent)
@staticmethod
def safe_cmp(val1, val2):
try:
return val1 == val2
except Exception:
return False
def sendWithHysteresis(self, intent):
if self._hysteresis_until.expired():
self._current_hysteresis = self._hysteresis_min_period
self._sender(intent)
else:
dups = self._keepIf(lambda M: (M.targetAddr != intent.targetAddr or
not HysteresisDelaySender.safe_cmp(
M.message, intent.message)))
# The dups are duplicate sends to the new intent's target;
# complete them when the actual message is finally sent
# with the same result
if dups:
intent.addCallback(self._dupSentGood(dups),
self._dupSentFail(dups))
self._hysteresis_queue.append(intent)
self._increase_hysteresis()
self._update_remaining_hysteresis_period()
def cancelSends(self, remoteAddr):
for each in self._keepIf(lambda M: M.targetAddr != remoteAddr):
each.tx_done(SendStatus.Failed)
def _keepIf(self, keepFunc):
requeues, removes = partition(keepFunc, self._hysteresis_queue)
self._hysteresis_queue = requeues
return removes
@staticmethod
def _dupSentGood(dups):
def _finishDups(result, finishedIntent):
for each in dups:
each.tx_done(result)
return _finishDups
@staticmethod
def _dupSentFail(dups):
def _finishDups(result, finishedIntent):
for each in dups:
each.tx_done(result)
return _finishDups
def drainTransmits(self):
drainLimit = ExpirationTimer(MAX_SHUTDOWN_DRAIN_PERIOD)
while not drainLimit.expired():
if not self.transport.run(TransmitOnly, drainLimit.remaining()):
break # no transmits left
