def run(self, incomingHandler, maximumDuration=None): """Core scheduling method; called by the current Actor process when idle to await new messages (or to do background processing). """ self._max_runtime = ExpiryTime(maximumDuration) # Always make at least one pass through to handle expired wakeups # and queued events; otherwise a null/negative maximumDuration could # block all processing. firstTime = True while firstTime or not self._max_runtime.expired(): firstTime = False now = datetime.now() self.run_time = min([ExpiryTime(P - now) for P in self._pendingWakeups] + [self._max_runtime]) rval = self._runWithExpiry(incomingHandler) if rval is not None: return rval if not self._realizeWakeups(): # No wakeups were processed, and the inner run # returned, so assume there's nothing to do and exit return rval while self._activeWakeups: w = self._activeWakeups.pop() if incomingHandler in (None, TransmitOnly): return w if not incomingHandler(w): return None return None
def testNoneComparedToZero(self): et1 = ExpiryTime(None) et2 = ExpiryTime(timedelta(days=0)) # None == forever, so it is greater than anything, although equal to itself assert et1 > et2 assert et2 < et1 assert et1 >= et2 assert et2 <= et1
def testNoneToUnExpiredComparison(self): et1 = ExpiryTime(None) et2 = ExpiryTime(timedelta(milliseconds=10)) assert et1 != et2 assert et2 != et1 sleep(et2.remainingSeconds()) assert et1 != et2 assert et2 != et1
def _update_remaining_hysteresis_period(self, reset=False): if not self._current_hysteresis: self._hysteresis_until = ExpiryTime(timedelta(seconds=0)) else: if reset or not self._hysteresis_until: self._hysteresis_until = ExpiryTime(self._current_hysteresis) else: self._hysteresis_until = ExpiryTime( self._current_hysteresis - self._hysteresis_until.remaining())
def check_convention(self): rmsgs = [] if not self._has_been_activated: return rmsgs if self.isConventionLeader() or not self.conventionLeaderAddr: missing = [ each for each in self._conventionMembers.values() if each.registryValid.expired() ] for each in missing: thesplog('%s missed %d checkins (%s); assuming it has died', str(each), CONVENTION_REGISTRATION_MISS_MAX, str(each.registryValid), level=logging.WARNING, primary=True) rmsgs.extend(self._remote_system_cleanup(each.remoteAddress)) self._conventionRegistration = ExpiryTime( CONVENTION_REREGISTRATION_PERIOD) else: # Re-register with the Convention if it's time if self.conventionLeaderAddr and self._conventionRegistration.expired( ): if getattr(self, '_conventionLeaderMissCount', 0) >= \ CONVENTION_REGISTRATION_MISS_MAX: thesplog( 'Admin convention registration lost @ %s (miss %d)', self.conventionLeaderAddr, self._conventionLeaderMissCount, level=logging.ERROR, primary=True) rmsgs.extend( self._remote_system_cleanup(self.conventionLeaderAddr)) self._conventionLeaderMissCount = 0 else: rmsgs.extend(self.setup_convention()) for member in self._conventionMembers.values(): if member.preRegistered and \ member.preRegistered.pingValid.expired() and \ not member.preRegistered.pingPending: member.preRegistered.pingPending = True # If remote misses a checkin, re-extend the # invitation. This also helps re-initiate a socket # connection if a TxOnly socket has been lost. member.preRegistered.pingValid = ExpiryTime( convention_reinvite_adjustment( CONVENTION_RESTART_PERIOD if member.registryValid. expired() else CONVENTION_REREGISTRATION_PERIOD)) rmsgs.append( HysteresisSend(member.remoteAddress, ConventionInvite(), onSuccess=self._preRegQueryNotPending, onError=self._preRegQueryNotPending)) return rmsgs
def testUnExpiredToUnExpiredComparison(self): et1 = ExpiryTime(timedelta(milliseconds=15)) et2 = ExpiryTime(timedelta(milliseconds=10)) assert et1 != et2 assert et2 != et1 sleep(et2.remainingSeconds()) print(str(et1), str(et2)) assert et1 != et2 assert et2 != et1 sleep(et1.remainingSeconds()) assert et1 == et2 assert et2 == et1
def testNoneComparedToNonZero(self): et1 = ExpiryTime(None) et2 = ExpiryTime(timedelta(milliseconds=10)) # None == forever, so it is greater than anything, although equal to itself assert et1 > et2 assert et2 < et1 assert et1 > et2 assert et2 < et1 sleep(et2.remainingSeconds()) assert et1 > et2 assert et2 < et1 assert et1 > et2 assert et2 < et1
def addWakeup(self, timePeriod): now = datetime.now() wakeupTime = now + timePeriod self._pendingWakeups.setdefault(wakeupTime, []) \ .append(ReceiveEnvelope(self.myAddress, WakeupMessage(timePeriod))) self.run_time = min([ExpiryTime(P - now) for P in self._pendingWakeups] + [self._max_runtime])
def tell(self, anActor, msg): attemptLimit = ExpiryTime(MAX_TELL_PERIOD) # transport may not use sockets, but this helps error handling # in case it does. import socket for attempt in range(5000): try: txwatch = self._tx_to_actor(anActor, msg) while not attemptLimit.expired(): if not self._run_transport(attemptLimit.remaining(), txonly=True): # all transmits completed return if txwatch.failed: raise ActorSystemFailure( 'Error sending to %s: %s' % (str(anActor), str(txwatch.failure))) raise ActorSystemRequestTimeout( 'Unable to send to %s within %s' % (str(anActor), str(MAX_CAPABILITY_UPDATE_DELAY))) except socket.error as ex: import errno if errno.EMFILE == ex.errno: import time time.sleep(0.1) else: raise
def testNonZeroIsFalse(self): et = ExpiryTime(timedelta(milliseconds=10)) assert not et assert not bool(et) sleep(et.remainingSeconds()) assert et assert bool(et)
def run(self): # Main loop for convention management. Wraps the lower-level # transport with a stop at the next needed convention # registration period to re-register. try: while not getattr(self, 'shutdown_completed', False): delay = min( self._cstate.convention_inattention_delay(), ExpiryTime(None) if self._hysteresisSender.delay.expired() else self._hysteresisSender.delay) # n.b. delay does not account for soon-to-expire # pingValids, but since delay will not be longer than # a CONVENTION_REREGISTRATION_PERIOD, the worst case # is a doubling of a pingValid period (which should be fine). r = self.transport.run(self.handleIncoming, delay.remaining()) # Check Convention status based on the elapsed time self._performIO(self._cstate.check_convention()) self._hysteresisSender.checkSends() except Exception as ex: import traceback thesplog('ActorAdmin uncaught exception: %s', traceback.format_exc(), level=logging.ERROR, exc_info=True) thesplog('Admin time to die', level=logging.DEBUG)
def ask(self, anActor, msg, timeout): txwatch = self._tx_to_actor(anActor, msg) # KWQ: pass timeout on tx?? askLimit = ExpiryTime(toTimeDeltaOrNone(timeout)) while not askLimit.expired(): response = self._run_transport(askLimit.remaining()) if txwatch.failed: if txwatch.failure in [SendStatus.DeadTarget, SendStatus.Failed, SendStatus.NotSent]: # Silent failure; not all transports can indicate # this, so for conformity the Dead Letter handler is # the intended method of handling this issue. return None raise ActorSystemFailure('Transmit of ask message to %s failed (%s)'%( str(anActor), str(txwatch.failure))) if response is None: # Timed out, give up. return None # Do not send miscellaneous ActorSystemMessages to the # caller that it might not recognize. If one of those was # recieved, loop to get another response. if response and \ hasattr(response, 'message') and \ not isInternalActorSystemMessage(response.message): return response.message return None
def loadActorSource(self, fname): self._LOADFAILED = None loadLimit = ExpiryTime(MAX_LOAD_SOURCE_DELAY) f = fname if hasattr(fname, 'read') else open(fname, 'rb') try: d = f.read() import hashlib hval = hashlib.md5(d).hexdigest() self.transport.scheduleTransmit( None, TransmitIntent( self.adminAddr, ValidateSource( hval, d, getattr( f, 'name', str(fname) if hasattr(fname, 'read') else fname)), onError=self._loadReqFailed)) while not loadLimit.expired(): if not self.transport.run(TransmitOnly, loadLimit.remaining()): break # all transmits completed if self._LOADFAILED or loadLimit.expired(): raise ActorSystemFailure('Load source failed due to ' + ( 'failure response (%s)' % self._LOADFAILED if self._LOADFAILED else 'timeout (%s)' % str(loadLimit))) return hval finally: f.close()
def setup_convention(self, activation=False): self._has_been_activated |= activation rmsgs = [] # If not specified in capabilities, don't override any invites # that may have been received. self._conventionAddress = self._getConventionAddr(self.capabilities) or \ self._conventionAddress leader_is_gone = (self._conventionMembers.find(self.conventionLeaderAddr) is None) \ if self.conventionLeaderAddr else True if not self.isConventionLeader() and self.conventionLeaderAddr: thesplog('Admin registering with Convention @ %s (%s)', self.conventionLeaderAddr, 'first time' if leader_is_gone else 're-registering', level=logging.INFO, primary=True) rmsgs.append( HysteresisSend(self.conventionLeaderAddr, ConventionRegister(self.myAddress, self.capabilities, leader_is_gone), onSuccess=self._setupConventionCBGood, onError=self._setupConventionCBError)) rmsgs.append(LogAggregator(self.conventionLeaderAddr)) self._conventionRegistration = ExpiryTime( CONVENTION_REREGISTRATION_PERIOD) return rmsgs
def shutdown(self): thesplog('ActorSystem shutdown requested.', level=logging.INFO) time_to_quit = ExpiryTime(MAX_SYSTEM_SHUTDOWN_DELAY) self.transport.scheduleTransmit( None, TransmitIntent(self.adminAddr, SystemShutdown(), onError=self._shutdownSendFailed)) while not time_to_quit.expired(): response = self.transport.run(None, time_to_quit.remaining()) if getattr(self, '_TASF', False): thesplog( 'Could not send shutdown request to Admin' '; aborting but not necessarily stopped', level=logging.WARNING) return if response: if isinstance(response.message, SystemShutdownCompleted): break else: thesplog('Expected shutdown completed message, got: %s', response.message, level=logging.WARNING) else: thesplog( 'No response to Admin shutdown request; Actor system not completely shutdown', level=logging.ERROR) thesplog('ActorSystem shutdown complete.')
def _reset_valid_timer(self): # registryValid is a timer that is usually set to a multiple # of the convention re-registration period. Each successful # convention re-registration resets the timer to the maximum # value (actually, it replaces this structure with a newly # generated structure). If the timer expires, the remote is # declared as dead and the registration is removed (or # quiesced if it is a pre-registration). self.registryValid = ExpiryTime(CONVENTION_REREGISTRATION_PERIOD * CONVENTION_REGISTRATION_MISS_MAX)
def __init__(self, actual_sender, hysteresis_min_period=HYSTERESIS_MIN_PERIOD, hysteresis_max_period=HYSTERESIS_MAX_PERIOD, hysteresis_rate=HYSTERESIS_RATE): self._sender = actual_sender self._hysteresis_until = ExpiryTime(timedelta(seconds=0)) self._hysteresis_queue = [] self._current_hysteresis = None # timedelta self._hysteresis_min_period = hysteresis_min_period self._hysteresis_max_period = hysteresis_max_period self._hysteresis_rate = hysteresis_rate
def unloadActorSource(self, sourceHash): loadLimit = ExpiryTime(MAX_LOAD_SOURCE_DELAY) txwatch = self._tx_to_admin(ValidateSource(sourceHash, None)) while not loadLimit.expired(): if not self._run_transport(loadLimit.remaining(), txonly=True): return # all transmits completed if txwatch.failed: raise ActorSystemFailure( 'Error sending source unload to Admin: %s' % str(txwatch.failure)) raise ActorSystemRequestTimeout('Unload source timeout: ' + str(loadLimit))
def updateCapability(self, capabilityName, capabilityValue=None): self._updCAPFAILED = False attemptLimit = ExpiryTime(MAX_CAPABILITY_UPDATE_DELAY) self.transport.scheduleTransmit( None, TransmitIntent(self.adminAddr, CapabilityUpdate(capabilityName, capabilityValue), onError=self._updateCapsFailed)) while not attemptLimit.expired(): if not self.transport.run(TransmitOnly, attemptLimit.remaining()): break # all transmits completed if self._updCAPFAILED or attemptLimit.expired(): raise ActorSystemFailure( "Could not update Actor System Admin capabilities.")
def __init__(self, myAddress, capabilities, sCBStats, getConventionAddressFunc): self._myAddress = myAddress self._capabilities = capabilities self._sCBStats = sCBStats self._conventionMembers = AssocList( ) # key=Remote Admin Addr, value=ConventionMemberData self._conventionNotificationHandlers = [] self._getConventionAddr = getConventionAddressFunc self._conventionAddress = getConventionAddressFunc(capabilities) self._conventionRegistration = ExpiryTime( CONVENTION_REREGISTRATION_PERIOD) self._has_been_activated = False self._invited = False # entered convention as a result of an explicit invite
def updateCapability(self, capabilityName, capabilityValue=None): attemptLimit = ExpiryTime(MAX_CAPABILITY_UPDATE_DELAY) txwatch = self._tx_to_admin(CapabilityUpdate(capabilityName, capabilityValue)) while not attemptLimit.expired(): if not self._run_transport(attemptLimit.remaining(), txonly=True): return # all transmits completed if txwatch.failed: raise ActorSystemFailure( 'Error sending capability updates to Admin: %s' % str(txwatch.failure)) raise ActorSystemRequestTimeout( 'Unable to confirm capability update in %s' % str(MAX_CAPABILITY_UPDATE_DELAY))
def unloadActorSource(self, sourceHash): self._LOADFAILED = None loadLimit = ExpiryTime(MAX_LOAD_SOURCE_DELAY) self.transport.scheduleTransmit( None, TransmitIntent(self.adminAddr, ValidateSource(sourceHash, None), onError=self._loadReqFailed)) while not loadLimit.expired(): if not self.transport.run(TransmitOnly, loadLimit.remaining()): break # all transmits completed if self._LOADFAILED or loadLimit.expired(): raise ActorSystemFailure('Unload source failed due to ' + ( 'failure response' if self._LOADFAILED else 'timeout (%s)' % str(loadLimit)))
def __init__(self, system, logDefs=None): self._numPrimaries = 0 # Expects self.transport has already been set by subclass __init__ self.adminAddr = self.transport.getAdminAddr(system.capabilities) tryingTime = ExpiryTime(MAX_SYSTEM_SHUTDOWN_DELAY + timedelta(seconds=1)) while not tryingTime.expired(): if not self.transport.probeAdmin(self.adminAddr): self._startAdmin(self.adminAddr, self.transport.myAddress, system.capabilities, logDefs) if self._verifyAdminRunning(): return import time time.sleep(0.5) # Previous version may have been exiting if not self._verifyAdminRunning(): raise InvalidActorAddress( self.adminAddr, 'not a valid or useable ActorSystem Admin')
def tell(self, anActor, msg): attemptLimit = ExpiryTime(MAX_TELL_PERIOD) import socket for attempt in range(5000): try: self.transport.scheduleTransmit( None, TransmitIntent(anActor, msg, onError=self._tellFailed)) while not attemptLimit.expired(): if not self.transport.run(TransmitOnly, attemptLimit.remaining()): break # all transmits completed return except socket.error as ex: import errno if errno.EMFILE == ex.errno: import time time.sleep(0.1) else: raise
def shutdown(self): thesplog('ActorSystem shutdown requested.', level=logging.INFO) time_to_quit = ExpiryTime(MAX_SYSTEM_SHUTDOWN_DELAY) txwatch = self._tx_to_admin(SystemShutdown()) while not time_to_quit.expired(): response = self._run_transport(time_to_quit.remaining()) if txwatch.failed: thesplog('Could not send shutdown request to Admin' '; aborting but not necessarily stopped', level=logging.WARNING) return if response: if isinstance(response.message, SystemShutdownCompleted): break else: thesplog('Expected shutdown completed message, got: %s', response.message, level=logging.WARNING) else: thesplog('No response to Admin shutdown request; Actor system not completely shutdown', level=logging.ERROR) self.transport.close() thesplog('ActorSystem shutdown complete.')
def loadActorSource(self, fname): loadLimit = ExpiryTime(MAX_LOAD_SOURCE_DELAY) f = fname if hasattr(fname, 'read') else open(fname, 'rb') try: d = f.read() import hashlib hval = hashlib.md5(d).hexdigest() txwatch = self._tx_to_admin( ValidateSource(hval, d, getattr(f, 'name', str(fname) if hasattr(fname, 'read') else fname))) while not loadLimit.expired(): if not self._run_transport(loadLimit.remaining(), txonly=True): # All transmits completed return hval if txwatch.failed: raise ActorSystemFailure( 'Error sending source load to Admin: %s' % str(txwatch.failure)) raise ActorSystemRequestTimeout('Load source timeout: ' + str(loadLimit)) finally: f.close()
def testNoneEquality(self): et1 = ExpiryTime(None) et2 = ExpiryTime(None) assert et1 == et2 assert et2 == et1
def testNoneToExpiredComparison(self): et1 = ExpiryTime(None) et2 = ExpiryTime(timedelta(minutes=0)) assert et1 != et2 assert et2 != et1
def testExpiredToExpiredComparison(self): et1 = ExpiryTime(timedelta(microseconds=0)) sleep(0.001) et2 = ExpiryTime(timedelta(minutes=0)) assert et1 == et2 assert et2 == et1
def drainTransmits(self): drainLimit = ExpiryTime(MAX_SHUTDOWN_DRAIN_PERIOD) while not drainLimit.expired(): if not self.transport.run(TransmitOnly, drainLimit.remaining()): break # no transmits left