def tell(self, anActor, msg): attemptLimit = ExpiryTime(MAX_TELL_PERIOD) # transport may not use sockets, but this helps error handling # in case it does. import socket for attempt in range(5000): try: txwatch = self._tx_to_actor(anActor, msg) while not attemptLimit.expired(): if not self._run_transport(attemptLimit.remaining(), txonly=True): # all transmits completed return if txwatch.failed: raise ActorSystemFailure( 'Error sending to %s: %s' % (str(anActor), str(txwatch.failure))) raise ActorSystemRequestTimeout( 'Unable to send to %s within %s' % (str(anActor), str(MAX_CAPABILITY_UPDATE_DELAY))) except socket.error as ex: import errno if errno.EMFILE == ex.errno: import time time.sleep(0.1) else: raise
def loadActorSource(self, fname): self._LOADFAILED = None loadLimit = ExpiryTime(MAX_LOAD_SOURCE_DELAY) f = fname if hasattr(fname, 'read') else open(fname, 'rb') try: d = f.read() import hashlib hval = hashlib.md5(d).hexdigest() self.transport.scheduleTransmit( None, TransmitIntent( self.adminAddr, ValidateSource( hval, d, getattr( f, 'name', str(fname) if hasattr(fname, 'read') else fname)), onError=self._loadReqFailed)) while not loadLimit.expired(): if not self.transport.run(TransmitOnly, loadLimit.remaining()): break # all transmits completed if self._LOADFAILED or loadLimit.expired(): raise ActorSystemFailure('Load source failed due to ' + ( 'failure response (%s)' % self._LOADFAILED if self._LOADFAILED else 'timeout (%s)' % str(loadLimit))) return hval finally: f.close()
def ask(self, anActor, msg, timeout): txwatch = self._tx_to_actor(anActor, msg) # KWQ: pass timeout on tx?? askLimit = ExpiryTime(toTimeDeltaOrNone(timeout)) while not askLimit.expired(): response = self._run_transport(askLimit.remaining()) if txwatch.failed: if txwatch.failure in [SendStatus.DeadTarget, SendStatus.Failed, SendStatus.NotSent]: # Silent failure; not all transports can indicate # this, so for conformity the Dead Letter handler is # the intended method of handling this issue. return None raise ActorSystemFailure('Transmit of ask message to %s failed (%s)'%( str(anActor), str(txwatch.failure))) if response is None: # Timed out, give up. return None # Do not send miscellaneous ActorSystemMessages to the # caller that it might not recognize. If one of those was # recieved, loop to get another response. if response and \ hasattr(response, 'message') and \ not isInternalActorSystemMessage(response.message): return response.message return None
def shutdown(self): thesplog('ActorSystem shutdown requested.', level=logging.INFO) time_to_quit = ExpiryTime(MAX_SYSTEM_SHUTDOWN_DELAY) self.transport.scheduleTransmit( None, TransmitIntent(self.adminAddr, SystemShutdown(), onError=self._shutdownSendFailed)) while not time_to_quit.expired(): response = self.transport.run(None, time_to_quit.remaining()) if getattr(self, '_TASF', False): thesplog( 'Could not send shutdown request to Admin' '; aborting but not necessarily stopped', level=logging.WARNING) return if response: if isinstance(response.message, SystemShutdownCompleted): break else: thesplog('Expected shutdown completed message, got: %s', response.message, level=logging.WARNING) else: thesplog( 'No response to Admin shutdown request; Actor system not completely shutdown', level=logging.ERROR) thesplog('ActorSystem shutdown complete.')
def unloadActorSource(self, sourceHash): loadLimit = ExpiryTime(MAX_LOAD_SOURCE_DELAY) txwatch = self._tx_to_admin(ValidateSource(sourceHash, None)) while not loadLimit.expired(): if not self._run_transport(loadLimit.remaining(), txonly=True): return # all transmits completed if txwatch.failed: raise ActorSystemFailure( 'Error sending source unload to Admin: %s' % str(txwatch.failure)) raise ActorSystemRequestTimeout('Unload source timeout: ' + str(loadLimit))
def updateCapability(self, capabilityName, capabilityValue=None): self._updCAPFAILED = False attemptLimit = ExpiryTime(MAX_CAPABILITY_UPDATE_DELAY) self.transport.scheduleTransmit( None, TransmitIntent(self.adminAddr, CapabilityUpdate(capabilityName, capabilityValue), onError=self._updateCapsFailed)) while not attemptLimit.expired(): if not self.transport.run(TransmitOnly, attemptLimit.remaining()): break # all transmits completed if self._updCAPFAILED or attemptLimit.expired(): raise ActorSystemFailure( "Could not update Actor System Admin capabilities.")
def updateCapability(self, capabilityName, capabilityValue=None): attemptLimit = ExpiryTime(MAX_CAPABILITY_UPDATE_DELAY) txwatch = self._tx_to_admin(CapabilityUpdate(capabilityName, capabilityValue)) while not attemptLimit.expired(): if not self._run_transport(attemptLimit.remaining(), txonly=True): return # all transmits completed if txwatch.failed: raise ActorSystemFailure( 'Error sending capability updates to Admin: %s' % str(txwatch.failure)) raise ActorSystemRequestTimeout( 'Unable to confirm capability update in %s' % str(MAX_CAPABILITY_UPDATE_DELAY))
def unloadActorSource(self, sourceHash): self._LOADFAILED = None loadLimit = ExpiryTime(MAX_LOAD_SOURCE_DELAY) self.transport.scheduleTransmit( None, TransmitIntent(self.adminAddr, ValidateSource(sourceHash, None), onError=self._loadReqFailed)) while not loadLimit.expired(): if not self.transport.run(TransmitOnly, loadLimit.remaining()): break # all transmits completed if self._LOADFAILED or loadLimit.expired(): raise ActorSystemFailure('Unload source failed due to ' + ( 'failure response' if self._LOADFAILED else 'timeout (%s)' % str(loadLimit)))
def tell(self, anActor, msg): attemptLimit = ExpiryTime(MAX_TELL_PERIOD) import socket for attempt in range(5000): try: self.transport.scheduleTransmit( None, TransmitIntent(anActor, msg, onError=self._tellFailed)) while not attemptLimit.expired(): if not self.transport.run(TransmitOnly, attemptLimit.remaining()): break # all transmits completed return except socket.error as ex: import errno if errno.EMFILE == ex.errno: import time time.sleep(0.1) else: raise
def shutdown(self): thesplog('ActorSystem shutdown requested.', level=logging.INFO) time_to_quit = ExpiryTime(MAX_SYSTEM_SHUTDOWN_DELAY) txwatch = self._tx_to_admin(SystemShutdown()) while not time_to_quit.expired(): response = self._run_transport(time_to_quit.remaining()) if txwatch.failed: thesplog('Could not send shutdown request to Admin' '; aborting but not necessarily stopped', level=logging.WARNING) return if response: if isinstance(response.message, SystemShutdownCompleted): break else: thesplog('Expected shutdown completed message, got: %s', response.message, level=logging.WARNING) else: thesplog('No response to Admin shutdown request; Actor system not completely shutdown', level=logging.ERROR) self.transport.close() thesplog('ActorSystem shutdown complete.')
def loadActorSource(self, fname): loadLimit = ExpiryTime(MAX_LOAD_SOURCE_DELAY) f = fname if hasattr(fname, 'read') else open(fname, 'rb') try: d = f.read() import hashlib hval = hashlib.md5(d).hexdigest() txwatch = self._tx_to_admin( ValidateSource(hval, d, getattr(f, 'name', str(fname) if hasattr(fname, 'read') else fname))) while not loadLimit.expired(): if not self._run_transport(loadLimit.remaining(), txonly=True): # All transmits completed return hval if txwatch.failed: raise ActorSystemFailure( 'Error sending source load to Admin: %s' % str(txwatch.failure)) raise ActorSystemRequestTimeout('Load source timeout: ' + str(loadLimit)) finally: f.close()
def testNonZeroRemaining(self): et = ExpiryTime(timedelta(milliseconds=10)) assert timedelta(days=0) < et.remaining() assert timedelta(milliseconds=11) > et.remaining() sleep(et.remainingSeconds()) assert timedelta(days=0) == et.remaining()
def testNoneRemaining(self): et = ExpiryTime(None) assert et.remaining() is None
def testNoneRemainingExplicitForever(self): et = ExpiryTime(None) assert 5 == et.remaining(5)
def testZeroRemaining(self): et = ExpiryTime(timedelta(seconds=0)) assert timedelta(days=0) == et.remaining()
def drainTransmits(self): drainLimit = ExpiryTime(MAX_SHUTDOWN_DRAIN_PERIOD) while not drainLimit.expired(): if not self.transport.run(TransmitOnly, drainLimit.remaining()): break # no transmits left
class HysteresisDelaySender(object): """Implements hysteresis delay for sending messages. This is intended to be used for messages exchanged between convention members to ensure that a mis-behaved member doesn't have the ability to inflict damage on the entire convention. The first time a message is sent via this sender it is passed on through, but that starts a blackout period that starts with the CONVENTION_HYSTERESIS_MIN_PERIOD. Each additional send attempt during that blackout period will cause the blackout period to be extended by the CONVENTION_HYSTERESIS_RATE, up to the CONVENTION_HYSTERESIS_MAX_PERIOD. Once the blackout period ends, the queued sends will be sent, but only the last attempted message of each type for the specified remote target. At that point, the hysteresis delay will be reduced by the CONVENTION_HYSTERESIS_RATE; further send attempts will affect the hysteresis blackout period as described as above but lack of sending attempts will continue to reduce the hysteresis back to a zero-delay setting. Note: delays are updated in a target-independent manner; the target is only considered when eliminating duplicates. Note: maxDelay on TransmitIntents is ignored by hysteresis delays. It is assumed that a transmit intent's maxDelay is greater than the maximum hysteresis period and/or that the hysteresis delay is more important than the transmit intent timeout. """ def __init__(self, actual_sender, hysteresis_min_period=HYSTERESIS_MIN_PERIOD, hysteresis_max_period=HYSTERESIS_MAX_PERIOD, hysteresis_rate=HYSTERESIS_RATE): self._sender = actual_sender self._hysteresis_until = ExpiryTime(timedelta(seconds=0)) self._hysteresis_queue = [] self._current_hysteresis = None # timedelta self._hysteresis_min_period = hysteresis_min_period self._hysteresis_max_period = hysteresis_max_period self._hysteresis_rate = hysteresis_rate @property def delay(self): return self._hysteresis_until def _has_hysteresis(self): return (self._current_hysteresis is not None and self._current_hysteresis >= self._hysteresis_min_period) def _increase_hysteresis(self): if self._has_hysteresis(): try: self._current_hysteresis = min( (self._current_hysteresis * self._hysteresis_rate), self._hysteresis_max_period) except TypeError: # See note below for _decrease_hysteresis self._current_hysteresis = min( timedelta(seconds=(self._current_hysteresis.seconds * self._hysteresis_rate)), self._hysteresis_max_period) else: self._current_hysteresis = self._hysteresis_min_period def _decrease_hysteresis(self): try: self._current_hysteresis = ((self._current_hysteresis / self._hysteresis_rate) if self._has_hysteresis() else None) except TypeError: # Python 2.x cannot multiply or divide a timedelta by a # fractional amount. There is also not a total_seconds # retrieval from a timedelta, but it should be safe to # assume that the hysteresis value is not greater than 1 # day. self._current_hysteresis = timedelta( seconds=(self._current_hysteresis.seconds / self._hysteresis_rate)) \ if self._has_hysteresis() else None def _update_remaining_hysteresis_period(self, reset=False): if not self._current_hysteresis: self._hysteresis_until = ExpiryTime(timedelta(seconds=0)) else: if reset or not self._hysteresis_until: self._hysteresis_until = ExpiryTime(self._current_hysteresis) else: self._hysteresis_until = ExpiryTime( self._current_hysteresis - self._hysteresis_until.remaining()) def checkSends(self): if self.delay.expired(): self._decrease_hysteresis() self._update_remaining_hysteresis_period(reset=True) for intent in self._keepIf(lambda M: False): self._sender(intent) @staticmethod def safe_cmp(val1, val2): try: return val1 == val2 except Exception: return False def sendWithHysteresis(self, intent): if self._hysteresis_until.expired(): self._current_hysteresis = self._hysteresis_min_period self._sender(intent) else: dups = self._keepIf(lambda M: (M.targetAddr != intent.targetAddr or not HysteresisDelaySender.safe_cmp( M.message, intent.message))) # The dups are duplicate sends to the new intent's target; # complete them when the actual message is finally sent # with the same result if dups: intent.addCallback(self._dupSentGood(dups), self._dupSentFail(dups)) self._hysteresis_queue.append(intent) self._increase_hysteresis() self._update_remaining_hysteresis_period() def cancelSends(self, remoteAddr): for each in self._keepIf(lambda M: M.targetAddr != remoteAddr): each.tx_done(SendStatus.Failed) def _keepIf(self, keepFunc): requeues, removes = partition(keepFunc, self._hysteresis_queue) self._hysteresis_queue = requeues return removes @staticmethod def _dupSentGood(dups): def _finishDups(result, finishedIntent): for each in dups: each.tx_done(result) return _finishDups @staticmethod def _dupSentFail(dups): def _finishDups(result, finishedIntent): for each in dups: each.tx_done(result) return _finishDups