def loadActorSource(self, fname):
self._LOADFAILED = None
loadLimit = ExpiryTime(MAX_LOAD_SOURCE_DELAY)
f = fname if hasattr(fname, 'read') else open(fname, 'rb')
try:
d = f.read()
import hashlib
hval = hashlib.md5(d).hexdigest()
self.transport.scheduleTransmit(
None,
TransmitIntent(
self.adminAddr,
ValidateSource(
hval, d,
getattr(
f, 'name',
str(fname) if hasattr(fname, 'read') else fname)),
onError=self._loadReqFailed))
while not loadLimit.expired():
if not self.transport.run(TransmitOnly, loadLimit.remaining()):
break # all transmits completed
if self._LOADFAILED or loadLimit.expired():
raise ActorSystemFailure('Load source failed due to ' + (
'failure response (%s)' %
self._LOADFAILED if self._LOADFAILED else 'timeout (%s)' %
str(loadLimit)))
return hval
finally:
f.close()
def shutdown(self):
thesplog('ActorSystem shutdown requested.', level=logging.INFO)
time_to_quit = ExpiryTime(MAX_SYSTEM_SHUTDOWN_DELAY)
self.transport.scheduleTransmit(
None,
TransmitIntent(self.adminAddr,
SystemShutdown(),
onError=self._shutdownSendFailed))
while not time_to_quit.expired():
response = self.transport.run(None, time_to_quit.remaining())
if getattr(self, '_TASF', False):
thesplog(
'Could not send shutdown request to Admin'
'; aborting but not necessarily stopped',
level=logging.WARNING)
return
if response:
if isinstance(response.message, SystemShutdownCompleted):
break
else:
thesplog('Expected shutdown completed message, got: %s',
response.message,
level=logging.WARNING)
else:
thesplog(
'No response to Admin shutdown request; Actor system not completely shutdown',
level=logging.ERROR)
thesplog('ActorSystem shutdown complete.')
def updateCapability(self, capabilityName, capabilityValue=None):
self._updCAPFAILED = False
attemptLimit = ExpiryTime(MAX_CAPABILITY_UPDATE_DELAY)
self.transport.scheduleTransmit(
None,
TransmitIntent(self.adminAddr,
CapabilityUpdate(capabilityName, capabilityValue),
onError = self._updateCapsFailed))
while not attemptLimit.expired():
if not self.transport.run(TransmitOnly, attemptLimit.remaining()):
break # all transmits completed
if self._updCAPFAILED or attemptLimit.expired():
raise ActorSystemFailure("Could not update Actor System Admin capabilities.")
def updateCapability(self, capabilityName, capabilityValue=None):
self._updCAPFAILED = False
attemptLimit = ExpiryTime(MAX_CAPABILITY_UPDATE_DELAY)
self.transport.scheduleTransmit(
None,
TransmitIntent(self.adminAddr,
CapabilityUpdate(capabilityName, capabilityValue),
onError = self._updateCapsFailed))
while not attemptLimit.expired():
if not self.transport.run(TransmitOnly, attemptLimit.remaining()):
break # all transmits completed
if self._updCAPFAILED or attemptLimit.expired():
raise ActorSystemFailure("Could not update Actor System Admin capabilities.")
def unloadActorSource(self, sourceHash):
self._LOADFAILED = None
loadLimit = ExpiryTime(MAX_LOAD_SOURCE_DELAY)
self.transport.scheduleTransmit(None,
TransmitIntent(self.adminAddr,
ValidateSource(sourceHash, None),
onError = self._loadReqFailed))
while not loadLimit.expired():
if not self.transport.run(TransmitOnly, loadLimit.remaining()):
break # all transmits completed
if self._LOADFAILED or loadLimit.expired():
raise ActorSystemFailure('Unload source failed due to ' +
('failure response' if self._LOADFAILED else
'timeout (%s)'%str(loadLimit)))
def unloadActorSource(self, sourceHash):
self._LOADFAILED = None
loadLimit = ExpiryTime(MAX_LOAD_SOURCE_DELAY)
self.transport.scheduleTransmit(None,
TransmitIntent(self.adminAddr,
ValidateSource(sourceHash, None),
onError = self._loadReqFailed))
while not loadLimit.expired():
if not self.transport.run(TransmitOnly, loadLimit.remaining()):
break # all transmits completed
if self._LOADFAILED or loadLimit.expired():
raise ActorSystemFailure('Unload source failed due to ' +
('failure response' if self._LOADFAILED else
'timeout (%s)'%str(loadLimit)))
def tell(self, anActor, msg):
attemptLimit = ExpiryTime(MAX_TELL_PERIOD)
import socket
for attempt in range(5000):
try:
self.transport.scheduleTransmit(
None,
TransmitIntent(anActor, msg, onError=self._tellFailed))
while not attemptLimit.expired():
if not self.transport.run(TransmitOnly, attemptLimit.remaining()):
break # all transmits completed
return
except socket.error as ex:
import errno
if errno.EMFILE == ex.errno:
import time
time.sleep(0.1)
else:
raise
def tell(self, anActor, msg):
attemptLimit = ExpiryTime(MAX_TELL_PERIOD)
import socket
for attempt in range(5000):
try:
self.transport.scheduleTransmit(
None,
TransmitIntent(anActor, msg, onError=self._tellFailed))
while not attemptLimit.expired():
if not self.transport.run(TransmitOnly, attemptLimit.remaining()):
break # all transmits completed
return
except socket.error as ex:
import errno
if errno.EMFILE == ex.errno:
import time
time.sleep(0.1)
else:
raise
def loadActorSource(self, fname):
self._LOADFAILED = None
loadLimit = ExpiryTime(MAX_LOAD_SOURCE_DELAY)
f = fname if hasattr(fname, 'read') else open(fname, 'rb')
try:
d = f.read()
import hashlib
hval = hashlib.md5(d).hexdigest()
self.transport.scheduleTransmit(None,
TransmitIntent(self.adminAddr,
ValidateSource(hval, d),
onError = self._loadReqFailed))
while not loadLimit.expired():
if not self.transport.run(TransmitOnly, loadLimit.remaining()):
break # all transmits completed
if self._LOADFAILED or loadLimit.expired():
raise ActorSystemFailure('Load source failed due to ' +
('failure response (%s)'%self._LOADFAILED
if self._LOADFAILED else
'timeout (%s)'%str(loadLimit)))
return hval
finally:
f.close()
def shutdown(self):
thesplog('ActorSystem shutdown requested.', level=logging.INFO)
time_to_quit = ExpiryTime(MAX_SYSTEM_SHUTDOWN_DELAY)
self.transport.scheduleTransmit(
None,
TransmitIntent(self.adminAddr, SystemShutdown(),
onError=self._shutdownSendFailed))
while not time_to_quit.expired():
response = self.transport.run(None, time_to_quit.remaining())
if getattr(self, '_TASF', False):
thesplog('Could not send shutdown request to Admin'
'; aborting but not necessarily stopped',
level=logging.WARNING)
return
if response:
if isinstance(response.message, SystemShutdownCompleted):
break
else:
thesplog('Expected shutdown completed message, got: %s', response.message,
level=logging.WARNING)
else:
thesplog('No response to Admin shutdown request; Actor system not completely shutdown',
level=logging.ERROR)
thesplog('ActorSystem shutdown complete.')
class HysteresisDelaySender(object):
"""Implements hysteresis delay for sending messages. This is intended
to be used for messages exchanged between convention members to
ensure that a mis-behaved member doesn't have the ability to
inflict damage on the entire convention. The first time a
message is sent via this sender it is passed on through, but
that starts a blackout period that starts with the
CONVENTION_HYSTERESIS_MIN_PERIOD. Each additional send attempt
during that blackout period will cause the blackout period to
be extended by the CONVENTION_HYSTERESIS_RATE, up to the
CONVENTION_HYSTERESIS_MAX_PERIOD. Once the blackout period
ends, the queued sends will be sent, but only the last
attempted message of each type for the specified remote target.
At that point, the hysteresis delay will be reduced by the
CONVENTION_HYSTERESIS_RATE; further send attempts will affect
the hysteresis blackout period as described as above but lack
of sending attempts will continue to reduce the hysteresis back
to a zero-delay setting.
Note: delays are updated in a target-independent manner; the
target is only considered when eliminating duplicates.
Note: maxDelay on TransmitIntents is ignored by hysteresis
delays. It is assumed that a transmit intent's maxDelay
is greater than the maximum hysteresis period and/or that
the hysteresis delay is more important than the transmit
intent timeout.
"""
def __init__(self, actual_sender,
hysteresis_min_period = HYSTERESIS_MIN_PERIOD,
hysteresis_max_period = HYSTERESIS_MAX_PERIOD,
hysteresis_rate = HYSTERESIS_RATE):
self._sender = actual_sender
self._hysteresis_until = ExpiryTime(timedelta(seconds=0))
self._hysteresis_queue = []
self._current_hysteresis = None # timedelta
self._hysteresis_min_period = hysteresis_min_period
self._hysteresis_max_period = hysteresis_max_period
self._hysteresis_rate = hysteresis_rate
@property
def delay(self):
return self._hysteresis_until
def _has_hysteresis(self):
return (self._current_hysteresis is not None and
self._current_hysteresis >= self._hysteresis_min_period)
def _increase_hysteresis(self):
self._current_hysteresis = min(
(self._current_hysteresis * self._hysteresis_rate)
if self._has_hysteresis() else self._hysteresis_min_period,
self._hysteresis_max_period)
def _decrease_hysteresis(self):
self._current_hysteresis = (
(self._current_hysteresis / self._hysteresis_rate)
if self._has_hysteresis() else None)
def _update_remaining_hysteresis_period(self, reset=False):
if not self._current_hysteresis:
self._hysteresis_until = ExpiryTime(timedelta(seconds=0))
else:
if reset or not self._hysteresis_until:
self._hysteresis_until = ExpiryTime(self._current_hysteresis)
else:
self._hysteresis_until = ExpiryTime(
self._current_hysteresis -
self._hysteresis_until.remaining())
def checkSends(self):
if self.delay.expired():
self._decrease_hysteresis()
self._update_remaining_hysteresis_period(reset=True)
for intent in self._keepIf(lambda M: False):
self._sender(intent)
def sendWithHysteresis(self, intent):
if self._hysteresis_until.expired():
self._current_hysteresis = self._hysteresis_min_period
self._sender(intent)
else:
dups = self._keepIf(lambda M: (M.targetAddr != intent.targetAddr or
type(M.message) != type(intent.message)))
# The dups are duplicate sends to the new intent's target; complete them when
# the actual message is finally sent with the same result
if dups:
intent.addCallback(self._dupSentGood(dups), self._dupSentFail(dups))
self._hysteresis_queue.append(intent)
self._increase_hysteresis()
self._update_remaining_hysteresis_period()
def cancelSends(self, remoteAddr):
for each in self._keepIf(lambda M: M.targetAddr != remoteAddr):
each.tx_done(SendStatus.Failed)
def _keepIf(self, keepFunc):
requeues, removes = partition(keepFunc, self._hysteresis_queue)
self._hysteresis_queue = requeues
return removes
@staticmethod
def _dupSentGood(dups):
def _finishDups(result, finishedIntent):
for each in dups:
each.tx_done(result)
return _finishDups
@staticmethod
def _dupSentFail(dups):
def _finishDups(result, finishedIntent):
for each in dups:
each.tx_done(result)
return _finishDups
def testNoneRemainingExplicitForever(self):
et = ExpiryTime(None)
self.assertEqual(5, et.remaining(5))
def testNonZeroRemaining(self):
et = ExpiryTime(timedelta(milliseconds=10))
self.assertTrue(timedelta(days=0) < et.remaining())
self.assertTrue(timedelta(milliseconds=11) > et.remaining())
sleep(et.remainingSeconds())
self.assertEqual(timedelta(days=0), et.remaining())
def testZeroRemaining(self):
et = ExpiryTime(timedelta(seconds=0))
self.assertEqual(timedelta(days=0), et.remaining())
def testNoneRemaining(self):
et = ExpiryTime(None)
self.assertIsNone(et.remaining())
def testNoneRemainingExplicitForever(self):
et = ExpiryTime(None)
assert 5 == et.remaining(5)
def testZeroRemaining(self):
et = ExpiryTime(timedelta(seconds=0))
assert timedelta(days=0) == et.remaining()
def testZeroRemaining(self):
et = ExpiryTime(timedelta(seconds=0))
self.assertEqual(timedelta(days=0), et.remaining())
def drainTransmits(self):
drainLimit = ExpiryTime(MAX_SHUTDOWN_DRAIN_PERIOD)
while not drainLimit.expired():
if not self.transport.run(TransmitOnly, drainLimit.remaining()):
break # no transmits left
def testNonZeroRemaining(self):
et = ExpiryTime(timedelta(milliseconds=10))
self.assertTrue(timedelta(days=0) < et.remaining())
self.assertTrue(timedelta(milliseconds=11) > et.remaining())
sleep(et.remainingSeconds())
self.assertEqual(timedelta(days=0), et.remaining())
def testNoneRemainingExplicitForever(self):
et = ExpiryTime(None)
self.assertEqual(5, et.remaining(5))
def testNoneRemaining(self):
et = ExpiryTime(None)
self.assertIsNone(et.remaining())
class HysteresisDelaySender(object):
"""Implements hysteresis delay for sending messages. This is intended
to be used for messages exchanged between convention members to
ensure that a mis-behaved member doesn't have the ability to
inflict damage on the entire convention. The first time a
message is sent via this sender it is passed on through, but
that starts a blackout period that starts with the
CONVENTION_HYSTERESIS_MIN_PERIOD. Each additional send attempt
during that blackout period will cause the blackout period to
be extended by the CONVENTION_HYSTERESIS_RATE, up to the
CONVENTION_HYSTERESIS_MAX_PERIOD. Once the blackout period
ends, the queued sends will be sent, but only the last
attempted message of each type for the specified remote target.
At that point, the hysteresis delay will be reduced by the
CONVENTION_HYSTERESIS_RATE; further send attempts will affect
the hysteresis blackout period as described as above but lack
of sending attempts will continue to reduce the hysteresis back
to a zero-delay setting.
Note: delays are updated in a target-independent manner; the
target is only considered when eliminating duplicates.
Note: maxDelay on TransmitIntents is ignored by hysteresis
delays. It is assumed that a transmit intent's maxDelay
is greater than the maximum hysteresis period and/or that
the hysteresis delay is more important than the transmit
intent timeout.
"""
def __init__(self, actual_sender,
hysteresis_min_period = HYSTERESIS_MIN_PERIOD,
hysteresis_max_period = HYSTERESIS_MAX_PERIOD,
hysteresis_rate = HYSTERESIS_RATE):
self._sender = actual_sender
self._hysteresis_until = ExpiryTime(timedelta(seconds=0))
self._hysteresis_queue = []
self._duplicate_queue = []
self._current_hysteresis = None # timedelta
self._hysteresis_min_period = hysteresis_min_period
self._hysteresis_max_period = hysteresis_max_period
self._hysteresis_rate = hysteresis_rate
@property
def delay(self): return self._hysteresis_until
def checkSends(self):
if self.delay.expired():
hsends = self._hysteresis_queue
self._hysteresis_queue = []
self._current_hysteresis = (
None
if (self._current_hysteresis is None or
self._current_hysteresis < self._hysteresis_min_period) else
self._current_hysteresis / self._hysteresis_rate)
self._hysteresis_until = ExpiryTime(self._current_hysteresis
if self._current_hysteresis else
timedelta(seconds=0))
for intent in hsends:
self._sender(intent)
def sendWithHysteresis(self, intent):
if self._hysteresis_until.expired():
self._current_hysteresis = self._hysteresis_min_period
self._sender(intent)
else:
dups = self._keepIf(lambda M: (M.targetAddr != intent.targetAddr or
type(M.message) != type(intent.message)))
# The dups are duplicate sends to the new intent's target; complete them when
# the actual message is finally sent with the same result
if dups:
intent.addCallback(self._dupSentGood(dups), self._dupSentFail(dups))
self._hysteresis_queue.append(intent)
self._current_hysteresis = min(
(self._hysteresis_min_period
if (self._current_hysteresis is None or
self._current_hysteresis < self._hysteresis_min_period) else
self._current_hysteresis * self._hysteresis_rate),
self._hysteresis_max_period)
self._hysteresis_until = ExpiryTime(
timedelta(seconds=0)
if not self._current_hysteresis else
(self._current_hysteresis -
(timedelta(seconds=0)
if not self._hysteresis_until else
self._hysteresis_until.remaining())))
def cancelSends(self, remoteAddr):
cancels = self._keepIf(lambda M: M.targetAddr != remoteAddr)
for each in cancels:
each.result = SendStatus.Failed
each.completionCallback()
def _keepIf(self, keepFunc):
requeues, removes = partition(keepFunc, self._hysteresis_queue)
self._hysteresis_queue = requeues
return removes
@staticmethod
def _dupSentGood(dups):
def _finishDups(result, finishedIntent):
for each in dups:
each.result = result
each.completionCallback()
return _finishDups
@staticmethod
def _dupSentFail(dups):
def _finishDups(result, finishedIntent):
for each in dups:
each.result = result
each.completionCallback()
return _finishDups
def testNonZeroRemaining(self):
et = ExpiryTime(timedelta(milliseconds=10))
assert timedelta(days=0) < et.remaining()
assert timedelta(milliseconds=11) > et.remaining()
sleep(et.remainingSeconds())
assert timedelta(days=0) == et.remaining()
def drainTransmits(self):
drainLimit = ExpiryTime(MAX_SHUTDOWN_DRAIN_PERIOD)
while not drainLimit.expired():
if not self.transport.run(TransmitOnly, drainLimit.remaining()):
break # no transmits left
def testNoneRemaining(self):
et = ExpiryTime(None)
assert et.remaining() is None