def run(self, incomingHandler, maximumDuration=None):
"""Core scheduling method; called by the current Actor process when
idle to await new messages (or to do background
processing).
"""
self._max_runtime = ExpiryTime(maximumDuration)
# Always make at least one pass through to handle expired wakeups
# and queued events; otherwise a null/negative maximumDuration could
# block all processing.
firstTime = True
while firstTime or not self._max_runtime.expired():
firstTime = False
now = datetime.now()
self.run_time = min([ExpiryTime(P - now) for P in self._pendingWakeups] +
[self._max_runtime])
rval = self._runWithExpiry(incomingHandler)
if rval is not None:
return rval
if not self._realizeWakeups():
# No wakeups were processed, and the inner run
# returned, so assume there's nothing to do and exit
return rval
while self._activeWakeups:
w = self._activeWakeups.pop()
if incomingHandler in (None, TransmitOnly):
return w
if not incomingHandler(w):
return None
return None
def testNoneComparedToZero(self):
et1 = ExpiryTime(None)
et2 = ExpiryTime(timedelta(days=0))
# None == forever, so it is greater than anything, although equal to itself
assert et1 > et2
assert et2 < et1
assert et1 >= et2
assert et2 <= et1
def testNoneToUnExpiredComparison(self):
et1 = ExpiryTime(None)
et2 = ExpiryTime(timedelta(milliseconds=10))
assert et1 != et2
assert et2 != et1
sleep(et2.remainingSeconds())
assert et1 != et2
assert et2 != et1
def _update_remaining_hysteresis_period(self, reset=False):
if not self._current_hysteresis:
self._hysteresis_until = ExpiryTime(timedelta(seconds=0))
else:
if reset or not self._hysteresis_until:
self._hysteresis_until = ExpiryTime(self._current_hysteresis)
else:
self._hysteresis_until = ExpiryTime(
self._current_hysteresis -
self._hysteresis_until.remaining())
def check_convention(self):
rmsgs = []
if not self._has_been_activated:
return rmsgs
if self.isConventionLeader() or not self.conventionLeaderAddr:
missing = [
each for each in self._conventionMembers.values()
if each.registryValid.expired()
]
for each in missing:
thesplog('%s missed %d checkins (%s); assuming it has died',
str(each),
CONVENTION_REGISTRATION_MISS_MAX,
str(each.registryValid),
level=logging.WARNING,
primary=True)
rmsgs.extend(self._remote_system_cleanup(each.remoteAddress))
self._conventionRegistration = ExpiryTime(
CONVENTION_REREGISTRATION_PERIOD)
else:
# Re-register with the Convention if it's time
if self.conventionLeaderAddr and self._conventionRegistration.expired(
):
if getattr(self, '_conventionLeaderMissCount', 0) >= \
CONVENTION_REGISTRATION_MISS_MAX:
thesplog(
'Admin convention registration lost @ %s (miss %d)',
self.conventionLeaderAddr,
self._conventionLeaderMissCount,
level=logging.ERROR,
primary=True)
rmsgs.extend(
self._remote_system_cleanup(self.conventionLeaderAddr))
self._conventionLeaderMissCount = 0
else:
rmsgs.extend(self.setup_convention())
for member in self._conventionMembers.values():
if member.preRegistered and \
member.preRegistered.pingValid.expired() and \
not member.preRegistered.pingPending:
member.preRegistered.pingPending = True
# If remote misses a checkin, re-extend the
# invitation. This also helps re-initiate a socket
# connection if a TxOnly socket has been lost.
member.preRegistered.pingValid = ExpiryTime(
convention_reinvite_adjustment(
CONVENTION_RESTART_PERIOD if member.registryValid.
expired() else CONVENTION_REREGISTRATION_PERIOD))
rmsgs.append(
HysteresisSend(member.remoteAddress,
ConventionInvite(),
onSuccess=self._preRegQueryNotPending,
onError=self._preRegQueryNotPending))
return rmsgs
def testUnExpiredToUnExpiredComparison(self):
et1 = ExpiryTime(timedelta(milliseconds=15))
et2 = ExpiryTime(timedelta(milliseconds=10))
assert et1 != et2
assert et2 != et1
sleep(et2.remainingSeconds())
print(str(et1), str(et2))
assert et1 != et2
assert et2 != et1
sleep(et1.remainingSeconds())
assert et1 == et2
assert et2 == et1
def testNoneComparedToNonZero(self):
et1 = ExpiryTime(None)
et2 = ExpiryTime(timedelta(milliseconds=10))
# None == forever, so it is greater than anything, although equal to itself
assert et1 > et2
assert et2 < et1
assert et1 > et2
assert et2 < et1
sleep(et2.remainingSeconds())
assert et1 > et2
assert et2 < et1
assert et1 > et2
assert et2 < et1
def addWakeup(self, timePeriod):
now = datetime.now()
wakeupTime = now + timePeriod
self._pendingWakeups.setdefault(wakeupTime, []) \
.append(ReceiveEnvelope(self.myAddress, WakeupMessage(timePeriod)))
self.run_time = min([ExpiryTime(P - now) for P in self._pendingWakeups] +
[self._max_runtime])
def tell(self, anActor, msg):
attemptLimit = ExpiryTime(MAX_TELL_PERIOD)
# transport may not use sockets, but this helps error handling
# in case it does.
import socket
for attempt in range(5000):
try:
txwatch = self._tx_to_actor(anActor, msg)
while not attemptLimit.expired():
if not self._run_transport(attemptLimit.remaining(),
txonly=True):
# all transmits completed
return
if txwatch.failed:
raise ActorSystemFailure(
'Error sending to %s: %s' % (str(anActor),
str(txwatch.failure)))
raise ActorSystemRequestTimeout(
'Unable to send to %s within %s' %
(str(anActor), str(MAX_CAPABILITY_UPDATE_DELAY)))
except socket.error as ex:
import errno
if errno.EMFILE == ex.errno:
import time
time.sleep(0.1)
else:
raise
def testNonZeroIsFalse(self):
et = ExpiryTime(timedelta(milliseconds=10))
assert not et
assert not bool(et)
sleep(et.remainingSeconds())
assert et
assert bool(et)
def run(self):
# Main loop for convention management. Wraps the lower-level
# transport with a stop at the next needed convention
# registration period to re-register.
try:
while not getattr(self, 'shutdown_completed', False):
delay = min(
self._cstate.convention_inattention_delay(),
ExpiryTime(None) if self._hysteresisSender.delay.expired()
else self._hysteresisSender.delay)
# n.b. delay does not account for soon-to-expire
# pingValids, but since delay will not be longer than
# a CONVENTION_REREGISTRATION_PERIOD, the worst case
# is a doubling of a pingValid period (which should be fine).
r = self.transport.run(self.handleIncoming, delay.remaining())
# Check Convention status based on the elapsed time
self._performIO(self._cstate.check_convention())
self._hysteresisSender.checkSends()
except Exception as ex:
import traceback
thesplog('ActorAdmin uncaught exception: %s',
traceback.format_exc(),
level=logging.ERROR,
exc_info=True)
thesplog('Admin time to die', level=logging.DEBUG)
def ask(self, anActor, msg, timeout):
txwatch = self._tx_to_actor(anActor, msg) # KWQ: pass timeout on tx??
askLimit = ExpiryTime(toTimeDeltaOrNone(timeout))
while not askLimit.expired():
response = self._run_transport(askLimit.remaining())
if txwatch.failed:
if txwatch.failure in [SendStatus.DeadTarget,
SendStatus.Failed,
SendStatus.NotSent]:
# Silent failure; not all transports can indicate
# this, so for conformity the Dead Letter handler is
# the intended method of handling this issue.
return None
raise ActorSystemFailure('Transmit of ask message to %s failed (%s)'%(
str(anActor),
str(txwatch.failure)))
if response is None:
# Timed out, give up.
return None
# Do not send miscellaneous ActorSystemMessages to the
# caller that it might not recognize. If one of those was
# recieved, loop to get another response.
if response and \
hasattr(response, 'message') and \
not isInternalActorSystemMessage(response.message):
return response.message
return None
def loadActorSource(self, fname):
self._LOADFAILED = None
loadLimit = ExpiryTime(MAX_LOAD_SOURCE_DELAY)
f = fname if hasattr(fname, 'read') else open(fname, 'rb')
try:
d = f.read()
import hashlib
hval = hashlib.md5(d).hexdigest()
self.transport.scheduleTransmit(
None,
TransmitIntent(
self.adminAddr,
ValidateSource(
hval, d,
getattr(
f, 'name',
str(fname) if hasattr(fname, 'read') else fname)),
onError=self._loadReqFailed))
while not loadLimit.expired():
if not self.transport.run(TransmitOnly, loadLimit.remaining()):
break # all transmits completed
if self._LOADFAILED or loadLimit.expired():
raise ActorSystemFailure('Load source failed due to ' + (
'failure response (%s)' %
self._LOADFAILED if self._LOADFAILED else 'timeout (%s)' %
str(loadLimit)))
return hval
finally:
f.close()
def setup_convention(self, activation=False):
self._has_been_activated |= activation
rmsgs = []
# If not specified in capabilities, don't override any invites
# that may have been received.
self._conventionAddress = self._getConventionAddr(self.capabilities) or \
self._conventionAddress
leader_is_gone = (self._conventionMembers.find(self.conventionLeaderAddr) is None) \
if self.conventionLeaderAddr else True
if not self.isConventionLeader() and self.conventionLeaderAddr:
thesplog('Admin registering with Convention @ %s (%s)',
self.conventionLeaderAddr,
'first time' if leader_is_gone else 're-registering',
level=logging.INFO,
primary=True)
rmsgs.append(
HysteresisSend(self.conventionLeaderAddr,
ConventionRegister(self.myAddress,
self.capabilities,
leader_is_gone),
onSuccess=self._setupConventionCBGood,
onError=self._setupConventionCBError))
rmsgs.append(LogAggregator(self.conventionLeaderAddr))
self._conventionRegistration = ExpiryTime(
CONVENTION_REREGISTRATION_PERIOD)
return rmsgs
def shutdown(self):
thesplog('ActorSystem shutdown requested.', level=logging.INFO)
time_to_quit = ExpiryTime(MAX_SYSTEM_SHUTDOWN_DELAY)
self.transport.scheduleTransmit(
None,
TransmitIntent(self.adminAddr,
SystemShutdown(),
onError=self._shutdownSendFailed))
while not time_to_quit.expired():
response = self.transport.run(None, time_to_quit.remaining())
if getattr(self, '_TASF', False):
thesplog(
'Could not send shutdown request to Admin'
'; aborting but not necessarily stopped',
level=logging.WARNING)
return
if response:
if isinstance(response.message, SystemShutdownCompleted):
break
else:
thesplog('Expected shutdown completed message, got: %s',
response.message,
level=logging.WARNING)
else:
thesplog(
'No response to Admin shutdown request; Actor system not completely shutdown',
level=logging.ERROR)
thesplog('ActorSystem shutdown complete.')
def _reset_valid_timer(self):
# registryValid is a timer that is usually set to a multiple
# of the convention re-registration period. Each successful
# convention re-registration resets the timer to the maximum
# value (actually, it replaces this structure with a newly
# generated structure). If the timer expires, the remote is
# declared as dead and the registration is removed (or
# quiesced if it is a pre-registration).
self.registryValid = ExpiryTime(CONVENTION_REREGISTRATION_PERIOD *
CONVENTION_REGISTRATION_MISS_MAX)
def __init__(self,
actual_sender,
hysteresis_min_period=HYSTERESIS_MIN_PERIOD,
hysteresis_max_period=HYSTERESIS_MAX_PERIOD,
hysteresis_rate=HYSTERESIS_RATE):
self._sender = actual_sender
self._hysteresis_until = ExpiryTime(timedelta(seconds=0))
self._hysteresis_queue = []
self._current_hysteresis = None # timedelta
self._hysteresis_min_period = hysteresis_min_period
self._hysteresis_max_period = hysteresis_max_period
self._hysteresis_rate = hysteresis_rate
def unloadActorSource(self, sourceHash):
loadLimit = ExpiryTime(MAX_LOAD_SOURCE_DELAY)
txwatch = self._tx_to_admin(ValidateSource(sourceHash, None))
while not loadLimit.expired():
if not self._run_transport(loadLimit.remaining(), txonly=True):
return # all transmits completed
if txwatch.failed:
raise ActorSystemFailure(
'Error sending source unload to Admin: %s' %
str(txwatch.failure))
raise ActorSystemRequestTimeout('Unload source timeout: ' +
str(loadLimit))
def updateCapability(self, capabilityName, capabilityValue=None):
self._updCAPFAILED = False
attemptLimit = ExpiryTime(MAX_CAPABILITY_UPDATE_DELAY)
self.transport.scheduleTransmit(
None,
TransmitIntent(self.adminAddr,
CapabilityUpdate(capabilityName, capabilityValue),
onError=self._updateCapsFailed))
while not attemptLimit.expired():
if not self.transport.run(TransmitOnly, attemptLimit.remaining()):
break # all transmits completed
if self._updCAPFAILED or attemptLimit.expired():
raise ActorSystemFailure(
"Could not update Actor System Admin capabilities.")
def __init__(self, myAddress, capabilities, sCBStats,
getConventionAddressFunc):
self._myAddress = myAddress
self._capabilities = capabilities
self._sCBStats = sCBStats
self._conventionMembers = AssocList(
) # key=Remote Admin Addr, value=ConventionMemberData
self._conventionNotificationHandlers = []
self._getConventionAddr = getConventionAddressFunc
self._conventionAddress = getConventionAddressFunc(capabilities)
self._conventionRegistration = ExpiryTime(
CONVENTION_REREGISTRATION_PERIOD)
self._has_been_activated = False
self._invited = False # entered convention as a result of an explicit invite
def updateCapability(self, capabilityName, capabilityValue=None):
attemptLimit = ExpiryTime(MAX_CAPABILITY_UPDATE_DELAY)
txwatch = self._tx_to_admin(CapabilityUpdate(capabilityName,
capabilityValue))
while not attemptLimit.expired():
if not self._run_transport(attemptLimit.remaining(), txonly=True):
return # all transmits completed
if txwatch.failed:
raise ActorSystemFailure(
'Error sending capability updates to Admin: %s' %
str(txwatch.failure))
raise ActorSystemRequestTimeout(
'Unable to confirm capability update in %s' %
str(MAX_CAPABILITY_UPDATE_DELAY))
def unloadActorSource(self, sourceHash):
self._LOADFAILED = None
loadLimit = ExpiryTime(MAX_LOAD_SOURCE_DELAY)
self.transport.scheduleTransmit(
None,
TransmitIntent(self.adminAddr,
ValidateSource(sourceHash, None),
onError=self._loadReqFailed))
while not loadLimit.expired():
if not self.transport.run(TransmitOnly, loadLimit.remaining()):
break # all transmits completed
if self._LOADFAILED or loadLimit.expired():
raise ActorSystemFailure('Unload source failed due to ' + (
'failure response' if self._LOADFAILED else 'timeout (%s)' %
str(loadLimit)))
def __init__(self, system, logDefs=None):
self._numPrimaries = 0
# Expects self.transport has already been set by subclass __init__
self.adminAddr = self.transport.getAdminAddr(system.capabilities)
tryingTime = ExpiryTime(MAX_SYSTEM_SHUTDOWN_DELAY +
timedelta(seconds=1))
while not tryingTime.expired():
if not self.transport.probeAdmin(self.adminAddr):
self._startAdmin(self.adminAddr, self.transport.myAddress,
system.capabilities, logDefs)
if self._verifyAdminRunning(): return
import time
time.sleep(0.5) # Previous version may have been exiting
if not self._verifyAdminRunning():
raise InvalidActorAddress(
self.adminAddr, 'not a valid or useable ActorSystem Admin')
def tell(self, anActor, msg):
attemptLimit = ExpiryTime(MAX_TELL_PERIOD)
import socket
for attempt in range(5000):
try:
self.transport.scheduleTransmit(
None, TransmitIntent(anActor,
msg,
onError=self._tellFailed))
while not attemptLimit.expired():
if not self.transport.run(TransmitOnly,
attemptLimit.remaining()):
break # all transmits completed
return
except socket.error as ex:
import errno
if errno.EMFILE == ex.errno:
import time
time.sleep(0.1)
else:
raise
def shutdown(self):
thesplog('ActorSystem shutdown requested.', level=logging.INFO)
time_to_quit = ExpiryTime(MAX_SYSTEM_SHUTDOWN_DELAY)
txwatch = self._tx_to_admin(SystemShutdown())
while not time_to_quit.expired():
response = self._run_transport(time_to_quit.remaining())
if txwatch.failed:
thesplog('Could not send shutdown request to Admin'
'; aborting but not necessarily stopped',
level=logging.WARNING)
return
if response:
if isinstance(response.message, SystemShutdownCompleted):
break
else:
thesplog('Expected shutdown completed message, got: %s', response.message,
level=logging.WARNING)
else:
thesplog('No response to Admin shutdown request; Actor system not completely shutdown',
level=logging.ERROR)
self.transport.close()
thesplog('ActorSystem shutdown complete.')
def loadActorSource(self, fname):
loadLimit = ExpiryTime(MAX_LOAD_SOURCE_DELAY)
f = fname if hasattr(fname, 'read') else open(fname, 'rb')
try:
d = f.read()
import hashlib
hval = hashlib.md5(d).hexdigest()
txwatch = self._tx_to_admin(
ValidateSource(hval, d, getattr(f, 'name',
str(fname)
if hasattr(fname, 'read')
else fname)))
while not loadLimit.expired():
if not self._run_transport(loadLimit.remaining(), txonly=True):
# All transmits completed
return hval
if txwatch.failed:
raise ActorSystemFailure(
'Error sending source load to Admin: %s' %
str(txwatch.failure))
raise ActorSystemRequestTimeout('Load source timeout: ' +
str(loadLimit))
finally:
f.close()
def testNoneEquality(self):
et1 = ExpiryTime(None)
et2 = ExpiryTime(None)
assert et1 == et2
assert et2 == et1
def testNoneToExpiredComparison(self):
et1 = ExpiryTime(None)
et2 = ExpiryTime(timedelta(minutes=0))
assert et1 != et2
assert et2 != et1
def testExpiredToExpiredComparison(self):
et1 = ExpiryTime(timedelta(microseconds=0))
sleep(0.001)
et2 = ExpiryTime(timedelta(minutes=0))
assert et1 == et2
assert et2 == et1
def drainTransmits(self):
drainLimit = ExpiryTime(MAX_SHUTDOWN_DRAIN_PERIOD)
while not drainLimit.expired():
if not self.transport.run(TransmitOnly, drainLimit.remaining()):
break # no transmits left