def setUp(self):
self.store = Store(stamp=0.0)
self.timer = StoreTimer(store=self.store, duration=1.0)
self.baseDirpath = tempfile.mkdtemp(prefix="raet", suffix="base", dir=TEMPDIR)
self.stack = None
# This has to be set to True in the only one process that would perform tearDown steps
self.engine = True
stacking.RoadStack.BurstSize = 100
def __init__(self,
store=None,
version=raeting.VERSION,
main=None,
puid=None,
local=None, #passed up from subclass
name='',
uid=None,
server=None,
ha=None,
bufcnt=2,
rxMsgs=None,
txMsgs=None,
rxes=None,
txes=None,
stats=None,
):
'''
Setup Stack instance
'''
self.store = store or Store(stamp=0.0)
self.version = version
self.main = main
if getattr(self, 'puid', None) is None:
self.puid = puid if puid is not None else self.Uid
self.local = local or lotting.Lot(stack=self,
name=name,
uid=uid,
ha=ha,)
self.local.stack = self
self.remotes = self.uidRemotes = odict() # remotes indexed by uid
self.nameRemotes = odict() # remotes indexed by name
self.bufcnt = bufcnt
if not server:
server = self.serverFromLocal()
self.server = server
if self.server:
if not self.server.reopen(): # open socket
raise raeting.StackError("Stack '{0}': Failed opening server at"
" '{1}'\n".format(self.name, self.server.ha))
self.ha = self.server.ha # update local host address after open
console.verbose("Stack '{0}': Opened server at '{1}'\n".format(self.name,
self.ha))
self.rxMsgs = rxMsgs if rxMsgs is not None else deque() # messages received
self.txMsgs = txMsgs if txMsgs is not None else deque() # messages to transmit
self.rxes = rxes if rxes is not None else deque() # udp packets received
self.txes = txes if txes is not None else deque() # udp packet to transmit
self.stats = stats if stats is not None else odict() # udp statistics
self.statTimer = StoreTimer(self.store)
def setUp(self):
self.store = Store(stamp=0.0)
self.timer = StoreTimer(store=self.store, duration=1.0)
self.baseDirpath=tempfile.mkdtemp(prefix="raet", suffix="base", dir=TEMPDIR)
stacking.RoadStack.Bk = raeting.BodyKind.json.value
#main stack
mainName = "main"
mainDirpath = os.path.join(self.baseDirpath, 'road', 'keep', mainName)
signer = nacling.Signer()
mainSignKeyHex = signer.keyhex
privateer = nacling.Privateer()
mainPriKeyHex = privateer.keyhex
#other stack
otherName = "other"
otherDirpath = os.path.join(self.baseDirpath, 'road', 'keep', otherName)
signer = nacling.Signer()
otherSignKeyHex = signer.keyhex
privateer = nacling.Privateer()
otherPriKeyHex = privateer.keyhex
keeping.clearAllKeep(mainDirpath)
keeping.clearAllKeep(otherDirpath)
self.main = stacking.RoadStack(store=self.store,
name=mainName,
main=True,
auto=raeting.AutoMode.once.value,
sigkey=mainSignKeyHex,
prikey=mainPriKeyHex,
dirpath=mainDirpath,
)
self.other = stacking.RoadStack(store=self.store,
name=otherName,
auto=raeting.AutoMode.once.value,
ha=("", raeting.RAET_TEST_PORT),
sigkey=otherSignKeyHex,
prikey=otherPriKeyHex,
dirpath=otherDirpath,
)
def setUp(self):
self.store = Store(stamp=0.0)
self.timer = StoreTimer(store=self.store, duration=1.0)
if sys.platform == 'win32':
self.tempDirpath = tempfile.mktemp(prefix="raet", suffix="base",
dir=TEMPDIR)
else:
self.tempDirpath = tempfile.mkdtemp(prefix="raet", suffix="base",
dir=TEMPDIR)
self.baseDirpath = os.path.join(self.tempDirpath, 'lane', 'keep')
# main stack
self.main = stacking.LaneStack(name='main',
uid=1,
lanename='cherry',
sockdirpath=self.baseDirpath)
#other stack
self.other = stacking.LaneStack(name='other',
uid=1,
lanename='cherry',
sockdirpath=self.baseDirpath)
def setUp(self):
self.store = storing.Store(stamp=0.0)
self.timer = StoreTimer(store=self.store, duration=1.0)
self.saltDirpath = tempfile.mkdtemp(prefix="salt",
suffix="main",
dir='/tmp')
pkiDirpath = os.path.join(self.saltDirpath, 'pki')
if not os.path.exists(pkiDirpath):
os.makedirs(pkiDirpath)
acceptedDirpath = os.path.join(pkiDirpath, 'accepted')
if not os.path.exists(acceptedDirpath):
os.makedirs(acceptedDirpath)
pendingDirpath = os.path.join(pkiDirpath, 'pending')
if not os.path.exists(pendingDirpath):
os.makedirs(pendingDirpath)
rejectedDirpath = os.path.join(pkiDirpath, 'rejected')
if not os.path.exists(rejectedDirpath):
os.makedirs(rejectedDirpath)
self.localFilepath = os.path.join(pkiDirpath, 'local.key')
if os.path.exists(self.localFilepath):
mode = os.stat(self.localFilepath).st_mode
os.chmod(self.localFilepath, mode | stat.S_IWUSR | stat.S_IWUSR)
self.cacheDirpath = os.path.join(self.saltDirpath, 'cache')
self.sockDirpath = os.path.join(self.saltDirpath, 'sock')
self.opts = dict(
__role='master',
id='master',
pki_dir=pkiDirpath,
sock_dir=self.sockDirpath,
cachedir=self.cacheDirpath,
open_mode=False,
auto_accept=True,
transport='raet',
)
self.mainKeeper = RaetKey(opts=self.opts)
self.baseDirpath = tempfile.mkdtemp(prefix="salt",
suffix="base",
dir='/tmp')
def streamGet():
"""
Create test server sent event stream that sends count events
"""
timer = StoreTimer(store, duration=2.0)
bottle.response.set_header('Content-Type',
'text/event-stream') #text
bottle.response.set_header('Cache-Control', 'no-cache')
# HTTP 1.1 servers detect text/event-stream and use Transfer-Encoding: chunked
# Set client-side auto-reconnect timeout, ms.
yield 'retry: 1000\n\n'
i = 0
yield 'id: {0}\n'.format(i)
i += 1
yield 'data: START\n\n'
n = 1
while not timer.expired:
yield 'id: {0}\n'.format(i)
i += 1
yield 'data: {0}\n\n'.format(n)
n += 1
yield "data: END\n\n"
class Stack(object):
'''
RAET protocol base stack object.
Should be subclassed for specific transport type such as UDP or UXD
'''
Count = 0
Uid = 0 # base for next unique id for local and remotes
def __init__(
self,
store=None,
version=raeting.VERSION,
main=None,
puid=None,
local=None, #passed up from subclass
name='',
uid=None,
server=None,
ha=None,
bufcnt=2,
rxMsgs=None,
txMsgs=None,
rxes=None,
txes=None,
stats=None,
):
'''
Setup Stack instance
'''
self.store = store or Store(stamp=0.0)
self.version = version
self.main = main
if getattr(self, 'puid', None) is None:
self.puid = puid if puid is not None else self.Uid
self.local = local or lotting.Lot(
stack=self,
name=name,
uid=uid,
ha=ha,
)
self.local.stack = self
self.remotes = self.uidRemotes = odict() # remotes indexed by uid
self.nameRemotes = odict() # remotes indexed by name
self.bufcnt = bufcnt
if not server:
server = self.serverFromLocal()
self.server = server
if self.server:
if not self.server.reopen(): # open socket
raise raeting.StackError(
"Stack '{0}': Failed opening server at"
" '{1}'\n".format(self.name, self.server.ha))
self.ha = self.server.ha # update local host address after open
console.verbose("Stack '{0}': Opened server at '{1}'\n".format(
self.name, self.ha))
self.rxMsgs = rxMsgs if rxMsgs is not None else deque(
) # messages received
self.txMsgs = txMsgs if txMsgs is not None else deque(
) # messages to transmit
self.rxes = rxes if rxes is not None else deque(
) # udp packets received
self.txes = txes if txes is not None else deque(
) # udp packet to transmit
self.stats = stats if stats is not None else odict() # udp statistics
self.statTimer = StoreTimer(self.store)
@property
def name(self):
'''
property that returns name of local interface
'''
return self.local.name
@name.setter
def name(self, value):
'''
setter for name property
'''
self.local.name = value
@property
def ha(self):
'''
property that returns host address
'''
return self.local.ha
@ha.setter
def ha(self, value):
self.local.ha = value
def serverFromLocal(self):
'''
Create server from local data
'''
return None
def nextUid(self):
'''
Generates next unique id number for local or remotes.
'''
self.puid += 1
if self.puid > long(0xffffffff):
self.puid = 1 # rollover to 1
return self.puid
def addRemote(self, remote):
'''
Add a remote to indexes
'''
uid = remote.uid
# allow for condition where local.uid == 0 and remote.uid == 0
if uid in self.uidRemotes or (uid and uid == self.local.uid):
emsg = "Cannot add remote at uid '{0}', alreadys exists".format(
uid)
raise raeting.StackError(emsg)
if remote.name in self.nameRemotes or remote.name == self.local.name:
emsg = "Cannot add remote at name '{0}', alreadys exists".format(
remote.name)
raise raeting.StackError(emsg)
remote.stack = self
self.uidRemotes[uid] = remote
self.nameRemotes[remote.name] = remote
return remote
def moveRemote(self, remote, new):
'''
Move remote at key remote.uid to new uid and replace the odict key index
so order is the same
'''
old = remote.uid
if new in self.uidRemotes or new == self.local.uid:
emsg = "Cannot move, remote to '{0}', already exists".format(new)
raise raeting.StackError(emsg)
if old not in self.uidRemotes:
emsg = "Cannot move remote at '{0}', does not exist".format(old)
raise raeting.StackError(emsg)
if remote is not self.uidRemotes[old]:
emsg = "Cannot move remote at '{0}', not identical".format(old)
raise raeting.StackError(emsg)
remote.uid = new
index = self.uidRemotes.keys().index(old)
del self.uidRemotes[old]
self.uidRemotes.insert(index, new, remote)
def renameRemote(self, remote, new):
'''
rename remote with old remote.name to new name but keep same index
'''
old = remote.name
if new != old:
if new in self.nameRemotes or new == self.local.name:
emsg = "Cannot rename remote to '{0}', already exists".format(
new)
raise raeting.StackError(emsg)
if old not in self.nameRemotes:
emsg = "Cannot rename remote '{0}', does not exist".format(old)
raise raeting.StackError(emsg)
if remote is not self.nameRemotes[old]:
emsg = "Cannot rename remote '{0}', not identical".format(old)
raise raeting.StackError(emsg)
remote.name = new
index = self.nameRemotes.keys().index(old)
del self.nameRemotes[old]
self.nameRemotes.insert(index, new, remote)
def removeRemote(self, remote):
'''
Remove remote from all remotes dicts
'''
uid = remote.uid
if uid not in self.uidRemotes:
emsg = "Cannot remove remote '{0}', does not exist".format(uid)
raise raeting.StackError(emsg)
if remote is not self.uidRemotes[uid]:
emsg = "Cannot remove remote '{0}', not identical".format(uid)
raise raeting.StackError(emsg)
del self.uidRemotes[uid]
del self.nameRemotes[remote.name]
def removeAllRemotes(self):
'''
Remove all the remotes
'''
remotes = self.remotes.values(
) #make copy since changing .remotes in-place
for remote in remotes:
self.removeRemote(remote)
def fetchUidByName(self, name):
'''
Search for remote with matching name
Return remote if found Otherwise return None
'''
remote = self.nameRemotes.get(name)
return (remote.uid if remote else None)
def incStat(self, key, delta=1):
'''
Increment stat key counter by delta
'''
if key in self.stats:
self.stats[key] += delta
else:
self.stats[key] = delta
def updateStat(self, key, value):
'''
Set stat key to value
'''
self.stats[key] = value
def clearStat(self, key):
'''
Set the specified state counter to zero
'''
if key in self.stats:
self.stats[key] = 0
def clearStats(self):
'''
Set all the stat counters to zero and reset the timer
'''
for key, value in self.stats.items():
self.stats[key] = 0
self.statTimer.restart()
def _handleOneReceived(self):
'''
Handle one received message from server
assumes that there is a server
'''
try:
rx, ra = self.server.receive() # if no data the duple is ('',None)
except socket.error as ex:
err = raeting.get_exception_error(ex)
if err == errno.ECONNRESET:
return False
if not rx: # no received data
return False
self.rxes.append((rx, ra)) # duple = ( packet, source address)
return True
def serviceReceives(self):
'''
Retrieve from server all recieved and put on the rxes deque
'''
if self.server:
while self._handleOneReceived():
pass
def serviceReceiveOnce(self):
'''
Retrieve from server one recieved and put on the rxes deque
'''
if self.server:
self._handleOneReceived()
def _handleOneRx(self):
'''
Handle on message from .rxes deque
Assumes that there is a message on the .rxes deque
'''
raw, sa = self.rxes.popleft()
console.verbose("{0} received raw message\n{1}\n".format(
self.name, raw))
processRx(packet=raw)
def serviceRxes(self):
'''
Process all messages in .rxes deque
'''
while self.rxes:
self._handleOneRx()
def serviceRxOnce(self):
'''
Process one messages in .rxes deque
'''
if self.rxes:
self._handleOneRx()
def processRx(self, packet):
'''
Process
'''
pass
def transmit(self, msg, uid=None):
'''
Append duple (msg, uid) to .txMsgs deque
If msg is not mapping then raises exception
If uid is None then it will default to the first entry in .remotes
'''
if not isinstance(msg, Mapping):
emsg = "Invalid msg, not a mapping {0}\n".format(msg)
console.terse(emsg)
self.incStat("invalid_transmit_body")
return
if uid is None:
if not self.remotes:
emsg = "No remote to send to\n"
console.terse(emsg)
self.incStat("invalid_destination")
return
uid = self.remotes.values()[0].uid
self.txMsgs.append((msg, uid))
def _handleOneTxMsg(self):
'''
Take one message from .txMsgs deque and handle it
Assumes there is a message on the deque
'''
body, uid = self.txMsgs.popleft() # duple (body dict, destination uid
self.message(body, uid=uid)
console.verbose("{0} sending\n{1}\n".format(self.name, body))
def serviceTxMsgs(self):
'''
Service .txMsgs queue of outgoing messages
'''
while self.txMsgs:
self._handleOneTxMsg()
def serviceTxMsgOnce(self):
'''
Service one message on .txMsgs queue of outgoing messages
'''
if self.txMsgs:
self._handleOneTxMsg()
def message(self, body, uid=None):
'''
Sends message body to remote at uid
'''
pass
def tx(self, packed, duid):
'''
Queue duple of (packed, da) on stack .txes queue
Where da is the ip destination (host,port) address associated with
the remote identified by duid
'''
if duid not in self.remotes:
msg = "Invalid destination remote id '{0}'".format(duid)
raise raeting.StackError(msg)
self.txes.append((packed, self.remotes[duid].ha))
def _handleOneTx(self, laters, blocks):
'''
Handle one message on .txes deque
Assumes there is a message
laters is deque of messages to try again later
blocks is list of destinations that already blocked on this service
'''
tx, ta = self.txes.popleft() # duple = (packet, destination address)
if ta in blocks: # already blocked on this iteration
laters.append((tx, ta)) # keep sequential
return
try:
self.server.send(tx, ta)
except socket.error as ex:
err = raeting.get_exception_error(ex)
errors = [
errno.EAGAIN, errno.EWOULDBLOCK, errno.ENETUNREACH,
errno.EHOSTUNREACH, errno.EHOSTDOWN, errno.ECONNRESET
]
if hasattr(errno, 'ETIME'):
errors.append(errno.ETIME)
if (err in errors):
# problem sending such as busy with last message. save it for later
laters.append((tx, ta))
blocks.append(ta)
else:
raise
def serviceTxes(self):
'''
Service the .txes deque to send messages through server
'''
if self.server:
laters = deque()
blocks = []
while self.txes:
self._handleOneTx(laters, blocks)
while laters:
self.txes.append(laters.popleft())
def serviceTxOnce(self):
'''
Service on message on the .txes deque to send through server
'''
if self.server:
laters = deque()
blocks = [] # will always be empty since only once
if self.txes:
self._handleOneTx(laters, blocks)
while laters:
self.txes.append(laters.popleft())
def serviceAllRx(self):
'''
Service:
server receive
rxes queue
process
'''
self.serviceReceives()
self.serviceRxes()
self.process()
def serviceAllTx(self):
'''
Service:
txMsgs queue
txes queue to server send
'''
self.serviceTxMsgs()
self.serviceTxes()
def serviceAll(self):
'''
Service or Process:
server receive
rxes queue
process
txMsgs queue
txes queue to server send
'''
self.serviceAllRx()
self.serviceAllTx()
def serviceServer(self):
'''
Service the server's receive and transmit queues
'''
self.serviceReceives()
self.serviceTxes()
def serviceOneAllRx(self):
'''
Propagate one packet all the way through the received side of the stack
Service:
server receive
rxes queue
process
'''
self.serviceReceiveOnce()
self.serviceRxOnce()
self.process()
def serviceOneAllTx(self):
'''
Propagate one packet all the way through the transmit side of the stack
Service:
txMsgs queue
txes queue to server send
'''
self.serviceTxMsgOnce()
self.serviceTxOnce()
def process(self):
'''
Allow timer based processing
'''
pass
def testValetServiceBottleStream(self):
"""
Test Valet WSGI service request response stream sse
"""
console.terse("{0}\n".format(self.testValetServiceBottleStream.__doc__))
try:
import bottle
except ImportError as ex:
console.terse("Bottle not available.\n")
return
store = storing.Store(stamp=0.0)
app = bottle.default_app() # create bottle app
@app.get('/stream')
def streamGet():
"""
Create test server sent event stream that sends count events
"""
timer = StoreTimer(store, duration=2.0)
bottle.response.set_header('Content-Type', 'text/event-stream') #text
bottle.response.set_header('Cache-Control', 'no-cache')
# HTTP 1.1 servers detect text/event-stream and use Transfer-Encoding: chunked
# Set client-side auto-reconnect timeout, ms.
yield 'retry: 1000\n\n'
i = 0
yield 'id: {0}\n'.format(i)
i += 1
yield 'data: START\n\n'
n = 1
while not timer.expired:
yield 'id: {0}\n'.format(i)
i += 1
yield 'data: {0}\n\n'.format(n)
n += 1
yield "data: END\n\n"
console.terse("{0}\n".format("Building Valet ...\n"))
wireLogAlpha = wiring.WireLog(buffify=True, same=True)
result = wireLogAlpha.reopen()
alpha = serving.Valet(port = 6101,
bufsize=131072,
wlog=wireLogAlpha,
store=store,
app=app)
self.assertIs(alpha.servant.reopen(), True)
self.assertEqual(alpha.servant.ha, ('0.0.0.0', 6101))
self.assertEqual(alpha.servant.eha, ('127.0.0.1', 6101))
console.terse("{0}\n".format("Building Patron ...\n"))
wireLogBeta = wiring.WireLog(buffify=True, same=True)
result = wireLogBeta.reopen()
path = "http://{0}:{1}/".format('localhost', alpha.servant.eha[1])
beta = clienting.Patron(bufsize=131072,
wlog=wireLogBeta,
store=store,
path=path,
reconnectable=True,
)
self.assertIs(beta.connector.reopen(), True)
self.assertIs(beta.connector.accepted, False)
self.assertIs(beta.connector.connected, False)
self.assertIs(beta.connector.cutoff, False)
request = odict([('method', u'GET'),
('path', u'/stream'),
('qargs', odict()),
('fragment', u''),
('headers', odict([('Accept', 'application/json'),
('Content-Length', 0)])),
('body', None),
])
beta.requests.append(request)
timer = StoreTimer(store, duration=1.0)
while (not timer.expired):
alpha.serviceAll()
time.sleep(0.05)
beta.serviceAll()
time.sleep(0.05)
store.advanceStamp(0.1)
self.assertIs(beta.connector.accepted, True)
self.assertIs(beta.connector.connected, True)
self.assertIs(beta.connector.cutoff, False)
self.assertEqual(len(alpha.servant.ixes), 1)
self.assertEqual(len(alpha.reqs), 1)
self.assertEqual(len(alpha.reps), 1)
requestant = alpha.reqs.values()[0]
self.assertEqual(requestant.method, request['method'])
self.assertEqual(requestant.url, request['path'])
self.assertEqual(requestant.headers, {'accept': 'application/json',
'accept-encoding': 'identity',
'content-length': '0',
'host': 'localhost:6101'})
#timed out while stream still open so no responses in .responses
self.assertIs(beta.waited, True)
self.assertIs(beta.respondent.ended, False)
self.assertEqual(len(beta.responses), 0)
self.assertIn('content-type', beta.respondent.headers)
self.assertEqual(beta.respondent.headers['content-type'], 'text/event-stream')
self.assertIn('transfer-encoding', beta.respondent.headers)
self.assertEqual(beta.respondent.headers['transfer-encoding'], 'chunked')
self.assertTrue(len(beta.events) >= 3)
self.assertEqual(beta.respondent.retry, 1000)
self.assertTrue(int(beta.respondent.leid) >= 2)
event = beta.events.popleft()
self.assertEqual(event, {'id': '0', 'name': '', 'data': 'START'})
event = beta.events.popleft()
self.assertEqual(event, {'id': '1', 'name': '', 'data': '1'})
event = beta.events.popleft()
self.assertEqual(event, {'id': '2', 'name': '', 'data': '2'})
beta.events.clear()
#keep going until ended
timer.restart(duration=1.5)
while (not timer.expired):
alpha.serviceAll()
time.sleep(0.05)
beta.serviceAll()
time.sleep(0.05)
store.advanceStamp(0.1)
self.assertTrue(len(beta.events) >= 3)
self.assertEqual(beta.respondent.leid, '9')
self.assertEqual(beta.events[-2], {'id': '9', 'name': '', 'data': '9'})
self.assertEqual(beta.events[-1], {'id': '9', 'name': '', 'data': 'END'})
beta.events.clear()
alpha.servant.closeAll()
beta.connector.close()
wireLogAlpha.close()
wireLogBeta.close()
def testValetServiceBottleSecure(self):
"""
Test Valet WSGI service secure TLS request response
"""
console.terse("{0}\n".format(
self.testValetServiceBottleSecure.__doc__))
try:
import bottle
except ImportError as ex:
console.terse("Bottle not available.\n")
return
store = storing.Store(stamp=0.0)
app = bottle.default_app() # create bottle app
@app.get('/echo')
@app.get('/echo/<action>')
@app.post('/echo')
@app.post('/echo/<action>')
def echoGet(action=None):
"""
Echo back request data
"""
query = dict(bottle.request.query.items())
body = bottle.request.json
raw = bottle.request.body.read()
form = odict(bottle.request.forms)
data = odict(verb=bottle.request.method,
url=bottle.request.url,
action=action,
query=query,
form=form,
content=body)
return data
console.terse("{0}\n".format("Building Valet ...\n"))
wireLogAlpha = wiring.WireLog(buffify=True, same=True)
result = wireLogAlpha.reopen()
serverCertCommonName = 'localhost' # match hostname uses servers's cert commonname
#serverKeypath = '/etc/pki/tls/certs/server_key.pem' # local server private key
#serverCertpath = '/etc/pki/tls/certs/server_cert.pem' # local server public cert
#clientCafilepath = '/etc/pki/tls/certs/client.pem' # remote client public cert
serverKeypath = self.certdirpath + '/server_key.pem' # local server private key
serverCertpath = self.certdirpath + '/server_cert.pem' # local server public cert
clientCafilepath = self.certdirpath + '/client.pem' # remote client public cert
alpha = serving.Valet(
port=6101,
bufsize=131072,
wlog=wireLogAlpha,
store=store,
app=app,
scheme='https',
keypath=serverKeypath,
certpath=serverCertpath,
cafilepath=clientCafilepath,
)
self.assertIs(alpha.servant.reopen(), True)
self.assertEqual(alpha.servant.ha, ('0.0.0.0', 6101))
self.assertEqual(alpha.servant.eha, ('127.0.0.1', 6101))
console.terse("{0}\n".format("Building Patron ...\n"))
wireLogBeta = wiring.WireLog(buffify=True, same=True)
result = wireLogBeta.reopen()
#clientKeypath = '/etc/pki/tls/certs/client_key.pem' # local client private key
#clientCertpath = '/etc/pki/tls/certs/client_cert.pem' # local client public cert
#serverCafilepath = '/etc/pki/tls/certs/server.pem' # remote server public cert
clientKeypath = self.certdirpath + '/client_key.pem' # local client private key
clientCertpath = self.certdirpath + '/client_cert.pem' # local client public cert
serverCafilepath = self.certdirpath + '/server.pem' # remote server public cert
path = "https://{0}:{1}/".format('localhost', alpha.servant.eha[1])
beta = clienting.Patron(bufsize=131072,
wlog=wireLogBeta,
store=store,
path=path,
reconnectable=True,
scheme='https',
certedhost=serverCertCommonName,
keypath=clientKeypath,
certpath=clientCertpath,
cafilepath=serverCafilepath)
self.assertIs(beta.connector.reopen(), True)
self.assertIs(beta.connector.accepted, False)
self.assertIs(beta.connector.connected, False)
self.assertIs(beta.connector.cutoff, False)
request = odict([
('method', u'GET'),
('path', u'/echo?name=fame'),
('qargs', odict()),
('fragment', u''),
('headers',
odict([('Accept', 'application/json'), ('Content-Length', 0)])),
])
beta.requests.append(request)
timer = StoreTimer(store, duration=1.0)
while (beta.requests or beta.connector.txes or not beta.responses
or not alpha.idle()):
alpha.serviceAll()
time.sleep(0.05)
beta.serviceAll()
time.sleep(0.05)
store.advanceStamp(0.1)
self.assertIs(beta.connector.accepted, True)
self.assertIs(beta.connector.connected, True)
self.assertIs(beta.connector.cutoff, False)
self.assertEqual(len(alpha.servant.ixes), 1)
self.assertEqual(len(alpha.reqs), 1)
self.assertEqual(len(alpha.reps), 1)
requestant = alpha.reqs.values()[0]
self.assertEqual(requestant.method, request['method'])
self.assertEqual(requestant.url, request['path'])
self.assertEqual(
requestant.headers, {
'accept': 'application/json',
'accept-encoding': 'identity',
'content-length': '0',
'host': 'localhost:6101'
})
self.assertEqual(len(beta.responses), 1)
response = beta.responses.popleft()
self.assertEqual(response['status'], 200)
self.assertEqual(response['reason'], 'OK')
self.assertEqual(response['body'], bytearray(b''))
self.assertEqual(
response['data'],
{
'action': None,
'content': None,
'form': {},
'query': {
'name': 'fame'
},
'url': 'https://localhost:6101/echo?name=fame',
'verb': 'GET'
},
)
responder = alpha.reps.values()[0]
self.assertTrue(responder.status.startswith, str(response['status']))
self.assertEqual(responder.headers, response['headers'])
alpha.servant.closeAll()
beta.connector.close()
wireLogAlpha.close()
wireLogBeta.close()
class BasicTestCase(unittest.TestCase):
""""""
def setUp(self):
self.store = Store(stamp=0.0)
self.timer = StoreTimer(store=self.store, duration=1.0)
self.baseDirpath=tempfile.mkdtemp(prefix="raet", suffix="base", dir=TEMPDIR)
stacking.RoadStack.Bk = raeting.BodyKind.json.value
#main stack
mainName = "main"
mainDirpath = os.path.join(self.baseDirpath, 'road', 'keep', mainName)
signer = nacling.Signer()
mainSignKeyHex = signer.keyhex
privateer = nacling.Privateer()
mainPriKeyHex = privateer.keyhex
#other stack
otherName = "other"
otherDirpath = os.path.join(self.baseDirpath, 'road', 'keep', otherName)
signer = nacling.Signer()
otherSignKeyHex = signer.keyhex
privateer = nacling.Privateer()
otherPriKeyHex = privateer.keyhex
keeping.clearAllKeep(mainDirpath)
keeping.clearAllKeep(otherDirpath)
self.main = stacking.RoadStack(store=self.store,
name=mainName,
main=True,
auto=raeting.AutoMode.once.value,
sigkey=mainSignKeyHex,
prikey=mainPriKeyHex,
dirpath=mainDirpath,
)
self.other = stacking.RoadStack(store=self.store,
name=otherName,
auto=raeting.AutoMode.once.value,
ha=("", raeting.RAET_TEST_PORT),
sigkey=otherSignKeyHex,
prikey=otherPriKeyHex,
dirpath=otherDirpath,
)
def tearDown(self):
self.main.server.close()
self.other.server.close()
self.main.clearAllDir()
self.other.clearAllDir()
if os.path.exists(self.baseDirpath):
shutil.rmtree(self.baseDirpath)
def join(self, timeout=None):
'''
Utility method to do join. Call from test method.
'''
console.terse("\nJoin Transaction **************\n")
if not self.other.remotes:
self.other.addRemote(estating.RemoteEstate(stack=self.other,
#name=self.main.local.name,
fuid=0, # vacuous join
sid=0, # always 0 for join
ha=self.main.local.ha)
)
self.other.join(timeout=timeout)
self.service()
def allow(self):
'''
Utility method to do allow. Call from test method.
'''
console.terse("\nAllow Transaction **************\n")
self.other.allow()
self.service()
def alive(self, initiator, correspondent):
'''
Utility method to do alive. Call from test method.
'''
console.terse("\nAlive Transaction **************\n")
initiator.alive()
self.service()
def service(self, duration=1.0, real=True):
'''
Utility method to service queues. Call from test method.
'''
self.timer.restart(duration=duration)
while not self.timer.expired:
self.other.serviceAll()
self.main.serviceAll()
if not (self.main.transactions or self.other.transactions):
break
self.store.advanceStamp(0.1)
if real:
time.sleep(0.1)
def serviceOther(self, duration=1.0, real=True):
'''
Utility method to only service other queues. Call from test method.
'''
self.timer.restart(duration=duration)
while not self.timer.expired:
self.other.serviceAll()
if not (self.other.transactions):
break
self.store.advanceStamp(0.1)
if real:
time.sleep(0.1)
def serviceMain(self, duration=1.0, real=True):
'''
Utility method to only service main queues. Call from test method.
'''
self.timer.restart(duration=duration)
while not self.timer.expired:
self.main.serviceAll()
if not (self.main.transactions):
break
self.store.advanceStamp(0.1)
if real:
time.sleep(0.1)
def bootstrap(self, bk=raeting.BodyKind.json.value):
'''
Initialize
main on port 7530 with uid of 1
other on port 7531 with uid of 1
Complete
main joined and allowed
other joined and allowed
'''
stacking.RoadStack.Bk = bk
self.assertEqual(self.main.name, 'main')
self.assertEqual(self.main.local.name, 'main')
self.assertEqual(self.main.ha, ("0.0.0.0", raeting.RAET_PORT))
self.assertEqual(self.other.name, 'other')
self.assertEqual(self.other.local.name, 'other')
self.assertEqual(self.other.ha, ("0.0.0.0", raeting.RAET_TEST_PORT))
self.join()
console.terse("\nStack '{0}' uid= {1}\n".format(self.main.name, self.main.local.uid))
self.assertEqual(self.main.local.uid, 1)
self.assertEqual(self.main.name, 'main')
self.assertEqual(len(self.main.transactions), 0)
remote = self.main.remotes.values()[0]
self.assertTrue(remote.joined)
self.assertEqual(remote.name, 'other')
self.assertEqual(remote.uid, 2)
self.assertEqual(remote.uid, remote.nuid)
self.assertEqual(remote.fuid, 2)
self.assertTrue(2 in self.main.remotes)
self.assertTrue(remote.uid in self.main.remotes)
self.assertTrue(len(self.main.remotes), 1)
self.assertTrue(len(self.main.nameRemotes), 1)
self.assertEqual(len(remote.transactions), 0)
self.assertTrue('other' in self.main.nameRemotes)
self.assertIs(self.main.nameRemotes[remote.name], remote)
console.terse("Stack '{0}' estate name '{1}' joined with '{2}' = {3}\n".format(
self.main.name, self.main.local.name, remote.name, remote.joined))
console.terse("\nStack '{0}' uid= {1}\n".format(self.other.name, self.other.local.uid))
self.assertEqual(self.other.local.uid, 1)
self.assertEqual(self.other.name, 'other')
self.assertEqual(len(self.other.transactions), 0)
remote = self.other.remotes.values()[0]
self.assertTrue(remote.joined)
self.assertEqual(remote.name, 'main')
self.assertEqual(remote.uid, 2)
self.assertEqual(remote.uid, remote.nuid)
self.assertEqual(remote.fuid, 2)
self.assertTrue(2 in self.other.remotes)
self.assertTrue(remote.uid in self.other.remotes)
self.assertTrue(len(self.other.remotes), 1)
self.assertTrue(len(self.other.nameRemotes), 1)
self.assertEqual(len(remote.transactions), 0)
self.assertTrue('main' in self.other.nameRemotes)
self.assertIs(self.other.nameRemotes[remote.name], remote)
console.terse("Stack '{0}' estate name '{1}' joined with '{2}' = {3}\n".format(
self.other.name, self.other.local.name, remote.name, remote.joined))
self.allow()
console.terse("\nStack '{0}' uid= {1}\n".format(self.main.name, self.main.local.uid))
self.assertEqual(len(self.main.transactions), 0)
remote = self.main.remotes.values()[0]
self.assertTrue(remote.allowed)
self.assertEqual(len(remote.transactions), 0)
console.terse("Stack '{0}' estate name '{1}' allowd with '{2}' = {3}\n".format(
self.main.name, self.main.local.name, remote.name, remote.allowed))
console.terse("\nStack '{0}' uid= {1}\n".format(self.other.name, self.other.local.uid))
self.assertEqual(len(self.other.transactions), 0)
remote = self.other.remotes.values()[0]
self.assertTrue(remote.allowed)
self.assertEqual(len(remote.transactions), 0)
console.terse("Stack '{0}' estate name '{1}' allowed with '{2}' = {3}\n".format(
self.other.name, self.other.local.name, remote.name, remote.allowed))
console.terse("\nMessage: other to main *********\n")
body = odict(what="This is a message to the main estate. How are you", extra="I am fine.")
self.other.txMsgs.append((body, self.other.remotes.values()[0].fuid, None))
#self.other.message(body=body, deid=self.main.local.uid)
self.service()
console.terse("\nStack '{0}' uid= {1}\n".format(self.main.name, self.main.local.uid))
self.assertEqual(len(self.main.transactions), 0)
for msg, name in self.main.rxMsgs:
console.terse("Estate '{0}' rxed:\n'{1}'\n".format(self.main.local.name, msg))
self.assertDictEqual(body, self.main.rxMsgs[0][0])
console.terse("\nMessage: main to other *********\n")
body = odict(what="This is a message to the other estate. Get to Work", extra="Fix the fence.")
self.main.txMsgs.append((body, self.main.remotes.values()[0].fuid, None))
#self.main.message(body=body, deid=self.other.local.uid)
self.service()
console.terse("\nStack '{0}' uid= {1}\n".format(self.other.name, self.other.local.uid))
self.assertEqual(len(self.other.transactions), 0)
for msg, name in self.other.rxMsgs:
console.terse("Estate '{0}' rxed:\n'{1}'\n".format(self.other.local.name, msg))
self.assertDictEqual(body, self.other.rxMsgs[0][0])
def bidirectional(self,
bk=raeting.BodyKind.json.value,
mains=None,
others=None,
duration=3.0,):
'''
Initialize
main on port 7530 with uid of 1
other on port 7531 with uid of 0
Complete
main uid of 1 joined and allowed
other uid of 2 joined and allowed
'''
stacking.RoadStack.Bk = bk
mains = mains or []
other = others or []
self.join()
self.assertEqual(len(self.main.transactions), 0)
remote = self.main.remotes.values()[0]
self.assertTrue(remote.joined)
self.assertEqual(len(self.other.transactions), 0)
remote = self.other.remotes.values()[0]
self.assertTrue(remote.joined)
self.allow()
self.assertEqual(len(self.main.transactions), 0)
remote = self.main.remotes.values()[0]
self.assertTrue(remote.allowed)
self.assertEqual(len(self.other.transactions), 0)
remote = self.other.remotes.values()[0]
self.assertTrue(remote.allowed)
console.terse("\nMessages Bidirectional *********\n")
for msg in mains:
self.main.transmit(msg)
for msg in others:
self.other.transmit(msg)
self.service(duration=duration)
console.terse("\nStack '{0}' uid= {1}\n".format(self.main.name, self.main.local.uid))
self.assertEqual(len(self.main.transactions), 0)
self.assertEqual(len(others), len(self.main.rxMsgs))
for i, duple in enumerate(self.main.rxMsgs):
console.terse("Estate '{0}' rxed:\n'{1}'\n".format(self.main.local.name, duple))
self.assertDictEqual(others[i], duple[0])
console.terse("\nStack '{0}' uid= {1}\n".format(self.other.name, self.other.local.uid))
self.assertEqual(len(self.other.transactions), 0)
self.assertEqual(len(mains), len(self.other.rxMsgs))
for i, duple in enumerate(self.other.rxMsgs):
console.terse("Estate '{0}' rxed:\n'{1}'\n".format(self.other.local.name, duple))
self.assertDictEqual(mains[i], duple[0])
def testBootstrapJson(self):
'''
Test join allow message transactions with JSON Serialization of body
'''
console.terse("{0}\n".format(self.testBootstrapJson.__doc__))
self.bootstrap(bk=raeting.BodyKind.json.value)
def testBootstrapMsgpack(self):
'''
Test join allow message transactions with MsgPack Serialization of body
'''
console.terse("{0}\n".format(self.testBootstrapMsgpack.__doc__))
self.bootstrap(bk=raeting.BodyKind.msgpack.value)
def testMsgBothwaysJson(self):
'''
Test message transactions
'''
console.terse("{0}\n".format(self.testMsgBothwaysJson.__doc__))
others = []
others.append(odict(house="Mama mia1", queue="fix me"))
others.append(odict(house="Mama mia2", queue="help me"))
others.append(odict(house="Mama mia3", queue="stop me"))
others.append(odict(house="Mama mia4", queue="run me"))
mains = []
mains.append(odict(house="Papa pia1", queue="fix me"))
mains.append(odict(house="Papa pia2", queue="help me"))
mains.append(odict(house="Papa pia3", queue="stop me"))
mains.append(odict(house="Papa pia4", queue="run me"))
self.bidirectional(bk=raeting.BodyKind.json.value, mains=mains, others=others)
def testMsgBothwaysMsgpack(self):
'''
Test message transactions
'''
console.terse("{0}\n".format(self.testMsgBothwaysMsgpack.__doc__))
others = []
others.append(odict(house="Mama mia1", queue="fix me"))
others.append(odict(house="Mama mia2", queue="help me"))
others.append(odict(house="Mama mia3", queue="stop me"))
others.append(odict(house="Mama mia4", queue="run me"))
mains = []
mains.append(odict(house="Papa pia1", queue="fix me"))
mains.append(odict(house="Papa pia2", queue="help me"))
mains.append(odict(house="Papa pia3", queue="stop me"))
mains.append(odict(house="Papa pia4", queue="run me"))
self.bidirectional(bk=raeting.BodyKind.msgpack.value, mains=mains, others=others)
def testMsgVeritive(self):
'''
Test message transactions where one is not veritive
'''
console.terse("{0}\n".format(self.testMsgVeritive.__doc__))
others = []
others.append(odict(house="Mama mia1", queue="fix me"))
others.append(odict(house="Mama mia2", queue="help me"))
others.append(odict(house="Mama mia3", queue="stop me"))
others.append(odict(house="Mama mia4", queue="run me"))
mains = []
mains.append(odict(house="Papa pia1", queue="fix me"))
mains.append(odict(house="Papa pia2", queue="help me"))
mains.append(odict(house="Papa pia3", queue="stop me"))
mains.append(odict(house="Papa pia4", queue="run me"))
self.assertIs(self.main.veritive, True)
self.assertIs(self.other.veritive, True)
self.bootstrap()
self.assertEqual(len(self.main.transactions), 0)
self.assertEqual(len(self.main.rxMsgs), 1)
self.main.rxMsgs.popleft()
self.assertEqual(len(self.other.transactions), 0)
self.assertEqual(len(self.other.rxMsgs), 1)
self.other.rxMsgs.popleft()
self.main.Fk = raeting.FootKind.nada.value # main does not sign its messages
msg = odict(house="Mama mia1", queue="fix me")
self.main.transmit(msg)
msg = odict(house="Papa pia1", queue="fix me")
self.other.transmit(msg)
self.service(duration=0.3)
# other will reject main's message because not signed
self.assertEqual(len(self.main.transactions), 1)
self.assertEqual(len(self.main.rxMsgs), 1)
self.main.rxMsgs.popleft()
self.assertEqual(len(self.other.transactions), 0)
self.assertEqual(len(self.other.rxMsgs), 0)
self.assertIn('parsing_outer_error', self.other.stats)
self.assertEqual(self.other.stats['parsing_outer_error'], 1)
remote = self.main.remotes.values()[0]
remote.transactions = odict() # clear transactions
self.assertEqual(len(self.main.transactions), 0) # no initated transaction
self.service(duration=0.25) # consume packet in UDP buffer
self.other.veritive = False # Other will accept unsigned messages
self.assertIs(self.other.veritive, False)
msg = odict(house="Mama mia1", queue="fix me")
self.main.transmit(msg)
msg = odict(house="Papa pia1", queue="fix me")
self.other.transmit(msg)
self.service(duration=0.3)
# other will accept main's unsigned message but other's done ack will not
# be accepted by main because it won't be signed and main's veritive is True
# so mains transaction does not complete
self.assertEqual(len(self.main.transactions), 1)
self.assertEqual(len(self.main.rxMsgs), 1)
self.main.rxMsgs.popleft()
self.assertEqual(len(self.other.transactions), 0)
self.assertEqual(len(self.other.rxMsgs), 1)
self.other.rxMsgs.popleft()
self.assertIn('parsing_outer_error', self.main.stats)
self.assertEqual(self.main.stats['parsing_outer_error'], 1)
remote = self.main.remotes.values()[0]
remote.transactions = odict() # clear transactions
self.assertEqual(len(self.main.transactions), 0) # no initated transaction
self.service(duration=0.5) # consume packets in UDP
self.main.veritive = False # main will accept unsigned messages
self.assertIs(self.main.veritive, False)
msg = odict(house="Mama mia1", queue="fix me")
self.main.transmit(msg)
msg = odict(house="Papa pia1", queue="fix me")
self.other.transmit(msg)
self.service(duration=0.3)
# main will now complete its transaction because it will accept unsigned acks
# other will accept unsigned but generates signed each message completes
self.assertEqual(len(self.main.transactions), 0)
self.assertEqual(len(self.main.rxMsgs), 1)
self.main.rxMsgs.popleft()
self.assertEqual(len(self.other.transactions), 0)
self.assertEqual(len(self.other.rxMsgs), 1)
self.other.rxMsgs.popleft()
def testSegmentedJson(self):
'''
Test segmented message transactions
'''
console.terse("{0}\n".format(self.testSegmentedJson.__doc__))
stuff = []
for i in range(300):
stuff.append(str(i).rjust(10, " "))
stuff = "".join(stuff)
others = []
mains = []
others.append(odict(house="Snake eyes", queue="near stuff", stuff=stuff))
mains.append(odict(house="Craps", queue="far stuff", stuff=stuff))
bloat = []
for i in range(300):
bloat.append(str(i).rjust(100, " "))
bloat = "".join(bloat)
others.append(odict(house="Other", queue="big stuff", bloat=bloat))
mains.append(odict(house="Main", queue="gig stuff", bloat=bloat))
self.bidirectional(bk=raeting.BodyKind.json.value, mains=mains, others=others, duration=20.0)
def testSegmentedMsgpack(self):
'''
Test segmented message transactions
'''
console.terse("{0}\n".format(self.testSegmentedMsgpack.__doc__))
stuff = []
for i in range(300):
stuff.append(str(i).rjust(10, " "))
stuff = "".join(stuff)
others = []
mains = []
others.append(odict(house="Snake eyes", queue="near stuff", stuff=stuff))
mains.append(odict(house="Craps", queue="far stuff", stuff=stuff))
bloat = []
for i in range(300):
bloat.append(str(i).rjust(100, " "))
bloat = "".join(bloat)
others.append(odict(house="Other", queue="big stuff", bloat=bloat))
mains.append(odict(house="Main", queue="gig stuff", bloat=bloat))
self.bidirectional(bk=raeting.BodyKind.msgpack.value, mains=mains, others=others, duration=20.0)
def testSegmentedJsonBurst(self):
'''
Test segmented message transactions with burst count limiting
'''
console.terse("{0}\n".format(self.testSegmentedJsonBurst.__doc__))
stuff = []
for i in range(300):
stuff.append(str(i).rjust(10, " "))
stuff = "".join(stuff)
others = []
mains = []
others.append(odict(house="Snake eyes", queue="near stuff", stuff=stuff))
mains.append(odict(house="Craps", queue="far stuff", stuff=stuff))
bloat = []
for i in range(300):
bloat.append(str(i).rjust(100, " "))
bloat = "".join(bloat)
others.append(odict(house="Other", queue="big stuff", bloat=bloat))
mains.append(odict(house="Main", queue="gig stuff", bloat=bloat))
stacking.RoadStack.BurstSize = 16
self.assertEqual(stacking.RoadStack.BurstSize, 16)
self.bidirectional(bk=raeting.BodyKind.json.value, mains=mains, others=others, duration=20.0)
stacking.RoadStack.BurstSize = 0
self.assertEqual(stacking.RoadStack.BurstSize, 0)
def testSegmentedMsgpackBurst(self):
'''
Test segmented message transactions with burst count limiting
'''
console.terse("{0}\n".format(self.testSegmentedMsgpackBurst.__doc__))
stuff = []
for i in range(300):
stuff.append(str(i).rjust(10, " "))
stuff = "".join(stuff)
others = []
mains = []
others.append(odict(house="Snake eyes", queue="near stuff", stuff=stuff))
mains.append(odict(house="Craps", queue="far stuff", stuff=stuff))
bloat = []
for i in range(300):
bloat.append(str(i).rjust(100, " "))
bloat = "".join(bloat)
others.append(odict(house="Other", queue="big stuff", bloat=bloat))
mains.append(odict(house="Main", queue="gig stuff", bloat=bloat))
stacking.RoadStack.BurstSize = 16
self.assertEqual(stacking.RoadStack.BurstSize, 16)
self.bidirectional(bk=raeting.BodyKind.msgpack.value, mains=mains, others=others, duration=20.0)
stacking.RoadStack.BurstSize = 0
self.assertEqual(stacking.RoadStack.BurstSize, 0)
def testJoinForever(self):
'''
Test other joining with timeout set to 0.0 and default
'''
console.terse("{0}\n".format(self.testJoinForever.__doc__))
self.other.addRemote(estating.RemoteEstate(stack=self.other,
fuid=0, # vacuous join
sid=0, # always 0 for join
ha=self.main.local.ha))
self.other.join(timeout=0.0) #attempt to join forever with timeout 0.0
self.serviceOther(duration=20.0, real=False) # only service other so no response
console.terse("\nStack '{0}' uid= {1}\n".format(self.main.name, self.main.local.uid))
self.assertEqual(self.main.local.uid, 1)
self.assertEqual(self.main.name, 'main')
self.assertEqual(len(self.main.transactions), 0)
self.assertEqual(len(self.main.remotes), 0)
console.terse("\nStack '{0}' uid= {1}\n".format(self.other.name, self.other.local.uid))
self.assertEqual(self.other.local.uid, 1)
self.assertEqual(self.other.name, 'other')
self.assertEqual(len(self.other.transactions), 1)
remote = self.other.remotes.values()[0]
self.assertIs(remote.joined, None)
self.assertEqual(remote.uid, 2)
console.terse("Stack '{0}' estate name '{1}' joined with '{2}' = {3}\n".format(
self.other.name, self.other.local.name, remote.name, remote.joined))
console.terse("{0} Stats\n".format(self.main.name))
for key, val in self.main.stats.items():
console.terse(" {0}={1}\n".format(key, val))
self.assertEqual(len(self.main.stats), 0)
console.terse("{0} Stats\n".format(self.other.name))
for key, val in self.other.stats.items():
console.terse(" {0}={1}\n".format(key, val))
self.assertEqual(self.other.stats.get('joiner_tx_join_redo'), 6)
# Now allow join to complete
self.service()
console.terse("\nStack '{0}' uid= {1}\n".format(self.main.name, self.main.local.uid))
self.assertEqual(self.main.local.uid, 1)
self.assertEqual(self.main.name, 'main')
self.assertEqual(len(self.main.transactions), 0)
remote = self.main.remotes.values()[0]
self.assertTrue(remote.joined)
self.assertEqual(remote.uid, 2)
self.assertTrue(2 in self.main.remotes)
self.assertTrue(len(self.main.remotes), 1)
self.assertTrue(len(self.main.nameRemotes), 1)
self.assertEqual(remote.name, 'other')
self.assertTrue('other' in self.main.nameRemotes)
console.terse("Stack '{0}' estate name '{1}' joined with '{2}' = {3}\n".format(
self.main.name, self.main.local.name, remote.name, remote.joined))
console.terse("\nStack '{0}' uid= {1}\n".format(self.other.name, self.other.local.uid))
self.assertEqual(self.other.local.uid, 1)
self.assertEqual(self.other.name, 'other')
self.assertEqual(len(self.other.transactions), 0)
remote = self.other.remotes.values()[0]
self.assertTrue(remote.joined)
self.assertEqual(remote.uid, 2)
self.assertTrue(2 in self.other.remotes)
self.assertTrue(len(self.other.remotes), 1)
self.assertTrue(len(self.other.nameRemotes), 1)
self.assertEqual(remote.name, 'main')
self.assertTrue('main' in self.other.nameRemotes)
console.terse("Stack '{0}' estate name '{1}' joined with '{2}' = {3}\n".format(
self.other.name, self.other.local.name, remote.name, remote.joined))
# Now try again with existing remote data
self.other.join(timeout=0.0) #attempt to join forever with timeout 0.0
self.serviceOther(duration=20.0, real=False) # only service other so no response
# main will still have join results from previous join transaction
console.terse("\nStack '{0}' uid= {1}\n".format(self.main.name, self.main.local.uid))
self.assertEqual(self.main.local.uid, 1)
self.assertEqual(self.main.name, 'main')
self.assertEqual(len(self.main.transactions), 0)
remote = self.main.remotes.values()[0]
self.assertTrue(remote.joined)
self.assertEqual(remote.uid, 2)
self.assertTrue(2 in self.main.remotes)
self.assertTrue(len(self.main.remotes), 1)
self.assertTrue(len(self.main.nameRemotes), 1)
self.assertEqual(remote.name, 'other')
self.assertTrue('other' in self.main.nameRemotes)
console.terse("Stack '{0}' estate name '{1}' joined with '{2}' = {3}\n".format(
self.main.name, self.main.local.name, remote.name, remote.joined))
# Other will have outstanding join transaction
console.terse("\nStack '{0}' uid= {1}\n".format(self.other.name, self.other.local.uid))
self.assertEqual(self.other.local.uid, 1)
self.assertEqual(self.other.name, 'other')
self.assertEqual(len(self.other.transactions), 1)
remote = self.other.remotes.values()[0]
self.assertIs(remote.joined, None)
self.assertEqual(remote.uid, 2)
self.assertTrue(2 in self.other.remotes)
self.assertTrue(len(self.other.remotes), 1)
self.assertTrue(len(self.other.nameRemotes), 1)
self.assertEqual(remote.name, 'main')
self.assertTrue('main' in self.other.nameRemotes)
console.terse("Stack '{0}' estate name '{1}' joined with '{2}' = {3}\n".format(
self.other.name, self.other.local.name, remote.name, remote.joined))
console.terse("{0} Stats\n".format(self.main.name))
for key, val in self.main.stats.items():
console.terse(" {0}={1}\n".format(key, val))
self.assertEqual(self.main.stats.get('join_correspond_complete'), 1)
console.terse("{0} Stats\n".format(self.other.name))
for key, val in self.other.stats.items():
console.terse(" {0}={1}\n".format(key, val))
self.assertEqual(self.other.stats.get('joiner_tx_join_redo'), 12)
self.assertEqual(self.other.stats.get('join_initiate_complete'), 1)
# Now allow join to complete
self.service()
console.terse("\nStack '{0}' uid= {1}\n".format(self.main.name, self.main.local.uid))
self.assertEqual(self.main.local.uid, 1)
self.assertEqual(self.main.name, 'main')
self.assertEqual(len(self.main.transactions), 0)
remote = self.main.remotes.values()[0]
self.assertTrue(remote.joined)
self.assertEqual(remote.uid, 2)
self.assertTrue(2 in self.main.remotes)
self.assertTrue(len(self.main.remotes), 1)
self.assertTrue(len(self.main.nameRemotes), 1)
self.assertEqual(remote.name, 'other')
self.assertTrue('other' in self.main.nameRemotes)
console.terse("Stack '{0}' estate name '{1}' joined with '{2}' = {3}\n".format(
self.main.name, self.main.local.name, remote.name, remote.joined))
console.terse("\nStack '{0}' uid= {1}\n".format(self.other.name, self.other.local.uid))
self.assertEqual(self.other.local.uid, 1)
self.assertEqual(self.other.name, 'other')
self.assertEqual(len(self.other.transactions), 0)
remote = self.other.remotes.values()[0]
self.assertTrue(remote.joined)
self.assertEqual(remote.uid, 2)
self.assertTrue(2 in self.other.remotes)
self.assertTrue(len(self.other.remotes), 1)
self.assertTrue(len(self.other.nameRemotes), 1)
self.assertEqual(remote.name, 'main')
self.assertTrue('main' in self.other.nameRemotes)
console.terse("Stack '{0}' estate name '{1}' joined with '{2}' = {3}\n".format(
self.other.name, self.other.local.name, remote.name, remote.joined))
def testStaleNack(self):
'''
Test stale nack
'''
console.terse("{0}\n".format(self.testStaleNack.__doc__))
self.join()
self.assertEqual(len(self.main.transactions), 0)
remote = self.main.remotes.values()[0]
self.assertTrue(remote.joined)
self.assertEqual(len(self.other.transactions), 0)
remote = self.other.remotes.values()[0]
self.assertTrue(remote.joined)
self.allow()
self.assertEqual(len(self.main.transactions), 0)
remote = self.main.remotes.values()[0]
self.assertTrue(remote.allowed)
self.assertEqual(len(self.other.transactions), 0)
remote = self.other.remotes.values()[0]
self.assertTrue(remote.allowed)
console.terse("\nMessage transaction *********\n")
body = odict(what="This is a message to the main estate. How are you", extra="I am fine.")
self.other.txMsgs.append((body, self.other.remotes.values()[0].fuid, None))
self.timer.restart(duration=1.0)
while not self.timer.expired:
self.other.serviceAllTx() # transmit but leave receives in socket buffer
self.main.serviceAllRx() # receive but leave transmits in queue
self.store.advanceStamp(0.1)
time.sleep(0.1)
self.assertEqual(len(self.main.transactions), 0) #completed
self.assertEqual(len(self.other.transactions), 1) # waiting for ack
remote = self.other.remotes.values()[0]
remote.transactions = odict() #clear transactions so RX is stale correspondent
self.assertEqual(len(self.other.transactions), 0) # no initated transaction
self.timer.restart(duration=2.0)
while not self.timer.expired:
self.main.serviceAll() # transmit stale ack
self.other.serviceAll() # recieve ack
self.store.advanceStamp(0.1)
time.sleep(0.1)
self.assertEqual(len(self.main.transactions), 0)
self.assertEqual(len(self.other.transactions), 0)
print("{0} Stats".format(self.main.name))
for key, val in self.main.stats.items():
print(" {0}={1}".format(key, val))
print()
print("{0} Stats".format(self.other.name))
for key, val in self.other.stats.items():
print(" {0}={1}".format(key, val))
print()
self.assertTrue(self.other.stats.get('stale_correspondent_attempt') >= 1)
self.assertTrue(self.other.stats.get('stale_correspondent_nack') >= 1)
self.assertTrue(self.main.stats.get('messengent_correspond_complete') >= 1)
self.assertTrue(self.main.stats.get('stale_packet') >= 1)
def testBasicAlive(self):
'''
Test basic alive transaction
'''
console.terse("{0}\n".format(self.testBasicAlive.__doc__))
self.join()
self.assertEqual(len(self.main.transactions), 0)
remote = self.main.remotes.values()[0]
self.assertTrue(remote.joined)
self.assertEqual(len(self.other.transactions), 0)
remote = self.other.remotes.values()[0]
self.assertTrue(remote.joined)
self.allow()
self.assertEqual(len(self.main.transactions), 0)
remote = self.main.remotes.values()[0]
self.assertTrue(remote.allowed)
self.assertTrue(remote.alived)
self.assertEqual(len(self.other.transactions), 0)
remote = self.other.remotes.values()[0]
self.assertTrue(remote.allowed)
self.assertTrue(remote.alived)
console.terse("\nAlive Other to Main *********\n")
otherRemote = self.main.remotes.values()[0]
mainRemote = self.other.remotes.values()[0]
otherRemote.alived = None
mainRemote.alived = None
self.alive(self.other, self.main)
self.assertEqual(len(self.main.transactions), 0)
self.assertTrue(otherRemote.alived)
self.assertEqual(len(self.other.transactions), 0)
self.assertTrue(mainRemote.alived)
console.terse("\nAlive Main to Other *********\n")
self.alive(self.main, self.other)
self.assertEqual(len(self.main.transactions), 0)
remote = self.main.remotes.values()[0]
self.assertTrue(remote.alived)
self.assertEqual(len(self.other.transactions), 0)
remote = self.other.remotes.values()[0]
self.assertTrue(remote.alived)
class RemoteEstate(Estate):
'''
RAET protocol endpoint remote estate object ie Remote Road Lot
Maintains verifier for verifying signatures and publican for encrypt/decrypt
.alived attribute is the dead or alive status of the remote
.alived = True, alive, recently have received valid signed packets from remote
.alive = False, dead, recently have not received valid signed packets from remote
.fuid is the far uid of the remote as owned by the farside stack
'''
def __init__(self,
stack,
uid=None,
fuid=0,
main=False,
kind=0,
verkey=None,
pubkey=None,
acceptance=None,
joined=None,
rsid=0,
**kwa):
'''
Setup instance
stack is required parameter
verkey is either nacl VerifyKey or raw or hex encoded key
pubkey is either nacl PublicKey or raw or hex encoded key
acceptance is accepted state of remote on Road
rsid is last received session id used by remotely initiated transaction
'''
if uid is None:
uid = stack.nextUid()
while uid in stack.remotes or uid == stack.local.uid:
uid = stack.nextUid()
if 'ha' not in kwa:
kwa['ha'] = ('127.0.0.1', raeting.RAET_TEST_PORT)
super(RemoteEstate, self).__init__(stack, uid=uid, **kwa)
self.fuid = fuid
self.main = main
self.kind = kind
self.joined = joined
self.allowed = None
self.alived = None
self.reaped = None
self.acceptance = acceptance
self.privee = nacling.Privateer() # short term key manager
self.publee = nacling.Publican(
) # correspondent short term key manager
self.verfer = nacling.Verifier(
verkey) # correspondent verify key manager
self.pubber = nacling.Publican(
pubkey) # correspondent long term key manager
self.rsid = rsid # last sid received from remote when RmtFlag is True
# persistence keep alive heartbeat timer. Initial duration has offset so
# not synced with other side persistence heatbeet
# by default do not use offset on main
if self.stack.main:
duration = self.stack.period
else:
duration = self.stack.period + self.stack.offset
self.timer = StoreTimer(store=self.stack.store, duration=duration)
self.reapTimer = StoreTimer(self.stack.store,
duration=self.stack.interim)
self.messages = deque(
) # deque of saved stale message body data to remote.uid
@property
def nuid(self):
'''
property that returns nuid, near uid, of remote as owned by nearside stack
alias for uid
'''
return self.uid
@nuid.setter
def nuid(self, value):
'''
setter for nuid, near uid, property
'''
self.uid = value
@property
def juid(self):
'''
property that returns juid, join uid, duple of (nuid, fuid)
nuid is near uid
fuid is far uid as owned by farside stack
'''
return (self.nuid, self.fuid)
@juid.setter
def juid(self, value):
'''
setter for juid, join uid, property, value is duple of (nuid, fuid)
'''
self.nuid, self.fuid = value
def rekey(self):
'''
Regenerate short term keys
'''
self.allowed = None
self.privee = nacling.Privateer() # short term key
self.publee = nacling.Publican(
) # correspondent short term key manager
def validRsid(self, rsid):
'''
Compare new rsid to old .rsid and return True
If old is zero Then new is always valid
If new is >= old modulo N where N is 2^32 = 0x100000000
And >= means the difference is less than N//2 = 0x80000000
(((new - old) % 0x100000000) < (0x100000000 // 2))
'''
return self.validateSid(new=rsid, old=self.rsid)
def refresh(self, alived=True):
'''
Restart presence heartbeat timer and conditionally reapTimer
If alived is None then do not change .alived but update timer
If alived is True then set .alived to True and handle implications
If alived is False the set .alived to False and handle implications
'''
self.timer.restart(duration=self.stack.period)
if alived is None:
return
if self.alived or alived: # alive before or after
self.reapTimer.restart()
if self.reaped:
self.unreap()
#otherwise let timer run both before and after are still dead
self.alived = alived
def manage(self, cascade=False, immediate=False):
'''
Perform time based processing of keep alive heatbeat
'''
if not self.reaped: # only manage alives if not already reaped
if immediate or self.timer.expired:
# alive transaction restarts self.timer
self.stack.alive(uid=self.uid, cascade=cascade)
if self.stack.interim > 0.0 and self.reapTimer.expired:
self.reap()
def reap(self):
'''
Remote is dead, reap it if main estate.
'''
if self.stack.main: # only main can reap
console.concise(
"Stack {0}: Reaping dead remote {1} at {2}\n".format(
self.stack.name, self.name, self.stack.store.stamp))
self.stack.incStat("remote_reap")
self.reaped = True
def unreap(self):
'''
Remote packet received from remote so not dead anymore.
'''
if self.stack.main: # only only main can reap or unreap
console.concise(
"Stack {0}: Unreaping dead remote {1} at {2}\n".format(
self.stack.name, self.name, self.stack.store.stamp))
self.stack.incStat("remote_unreap")
self.reaped = False
def removeStaleCorrespondents(self):
'''
Remove local stale correspondent transactions associated with remote
Local correspondent is indicated by rf ==True
Stale means the sid in the transaction is older than the current .rsid
assuming neither is zero, that is sid in index is older than remote.rsid
old rsid == 0 means new always valid
When sid in index is older than remote.rsid
Where index is tuple: (rf, le, re, si, ti, bf,)
rf = Remotely Initiated Flag, RmtFlag
le = leid, Local estate ID, LEID
re = reid, Remote estate ID, REID
si = sid, Session ID, SID
ti = tid, Transaction ID, TID
bf = Broadcast Flag, BcstFlag
'''
for index, transaction in self.transactions.items():
sid = index[3]
rf = index[0] # correspondent
if rf and not self.validRsid(sid):
transaction.nack()
self.removeTransaction(index)
emsg = (
"Stack {0}: Stale correspondent {1} from remote {1} at {2}"
"\n".format(self.stack.name, index, self.name,
self.stack.store.stamp))
console.terse(emsg)
self.stack.incStat('stale_correspondent')
self.doneTransactions.clear()
def replaceStaleInitiators(self):
'''
Save and remove any messages from messenger transactions initiated locally
with remote
Remove non message stale local initiator transactions associated with remote
Also save and requeue any stale locally initiated message transactions.
Local inititors have rf flag == False
Stale means the sid in the transaction is older than the current .sid
assuming neither is zero, that is sid in index is older than remote.sid
old sid == 0 means new always valid
Where index is tuple: (rf, le, re, si, ti, bf,)
rf = Remotely Initiated Flag, RmtFlag
le = leid, Local estate ID, LEID
re = reid, Remote estate ID, REID
si = sid, Session ID, SID
ti = tid, Transaction ID, TID
bf = Broadcast Flag, BcstFlag
'''
for index, transaction in self.transactions.items():
rf = index[0]
sid = index[3]
if not rf and not self.validSid(
sid): # transaction sid newer or equal
if transaction.kind in [TrnsKind.message]:
self.saveMessage(transaction)
transaction.nack()
self.removeTransaction(index)
emsg = ("Stack {0}: Stale initiator {1} to remote {2} at {3}"
"\n".format(self.stack.name, index, self.name,
self.stack.store.stamp))
console.terse(emsg)
self.stack.incStat('stale_initiator')
def saveMessage(self, messenger):
'''
Save copy of body data from stale initiated messenger onto .messages deque
for retransmitting later after new session is established
messenger is instance of Messenger compatible transaction
'''
self.messages.append(odict(messenger.tray.body))
emsg = ("Stack {0}: Saved stale message with remote {1}"
"\n".format(self.stack.name, self.name))
console.concise(emsg)
def sendSavedMessages(self):
'''
Message is Messenger compatible transaction
Save stale initiated message for retransmitting later after new session is established
'''
while self.messages:
body = self.messages.popleft()
self.stack.message(body, uid=self.uid)
emsg = ("Stack {0}: Resent saved message with remote {1}"
"\n".format(self.stack.name, self.name))
console.concise(emsg)
def allowInProcess(self):
'''
Returns list of transactions for all allow transactions with this remote
that are already in process
'''
return ([
t for t in self.transactions.values() if t.kind == TrnsKind.allow
])
def joinInProcess(self):
'''
Returns list of transactions for all join transaction with this remote
that are already in process
'''
return ([
t for t in self.transactions.values() if t.kind == TrnsKind.join
])
class Stack(object):
'''
RAET protocol base stack object.
Should be subclassed for specific transport type such as UDP or UXD
'''
Count = 0
Uid = 0 # base for next unique id for local and remotes
def __init__(self,
store=None,
version=raeting.VERSION,
main=None,
puid=None,
local=None, #passed up from subclass
name='',
uid=None,
server=None,
ha=None,
bufcnt=2,
rxMsgs=None,
txMsgs=None,
rxes=None,
txes=None,
stats=None,
):
'''
Setup Stack instance
'''
self.store = store or Store(stamp=0.0)
self.version = version
self.main = main
if getattr(self, 'puid', None) is None:
self.puid = puid if puid is not None else self.Uid
self.local = local or lotting.Lot(stack=self,
name=name,
uid=uid,
ha=ha,)
self.local.stack = self
self.remotes = self.uidRemotes = odict() # remotes indexed by uid
self.nameRemotes = odict() # remotes indexed by name
self.bufcnt = bufcnt
if not server:
server = self.serverFromLocal()
self.server = server
if self.server:
if not self.server.reopen(): # open socket
raise raeting.StackError("Stack '{0}': Failed opening server at"
" '{1}'\n".format(self.name, self.server.ha))
self.ha = self.server.ha # update local host address after open
console.verbose("Stack '{0}': Opened server at '{1}'\n".format(self.name,
self.ha))
self.rxMsgs = rxMsgs if rxMsgs is not None else deque() # messages received
self.txMsgs = txMsgs if txMsgs is not None else deque() # messages to transmit
self.rxes = rxes if rxes is not None else deque() # udp packets received
self.txes = txes if txes is not None else deque() # udp packet to transmit
self.stats = stats if stats is not None else odict() # udp statistics
self.statTimer = StoreTimer(self.store)
@property
def name(self):
'''
property that returns name of local interface
'''
return self.local.name
@name.setter
def name(self, value):
'''
setter for name property
'''
self.local.name = value
@property
def ha(self):
'''
property that returns host address
'''
return self.local.ha
@ha.setter
def ha(self, value):
self.local.ha = value
def serverFromLocal(self):
'''
Create server from local data
'''
return None
def nextUid(self):
'''
Generates next unique id number for local or remotes.
'''
self.puid += 1
if self.puid > long(0xffffffff):
self.puid = 1 # rollover to 1
return self.puid
def addRemote(self, remote):
'''
Add a remote to indexes
'''
uid = remote.uid
# allow for condition where local.uid == 0 and remote.uid == 0
if uid in self.uidRemotes or (uid and uid == self.local.uid):
emsg = "Cannot add remote at uid '{0}', alreadys exists".format(uid)
raise raeting.StackError(emsg)
if remote.name in self.nameRemotes or remote.name == self.local.name:
emsg = "Cannot add remote at name '{0}', alreadys exists".format(remote.name)
raise raeting.StackError(emsg)
remote.stack = self
self.uidRemotes[uid] = remote
self.nameRemotes[remote.name] = remote
return remote
def moveRemote(self, remote, new):
'''
Move remote at key remote.uid to new uid and replace the odict key index
so order is the same
'''
old = remote.uid
if new in self.uidRemotes or new == self.local.uid:
emsg = "Cannot move, remote to '{0}', already exists".format(new)
raise raeting.StackError(emsg)
if old not in self.uidRemotes:
emsg = "Cannot move remote at '{0}', does not exist".format(old)
raise raeting.StackError(emsg)
if remote is not self.uidRemotes[old]:
emsg = "Cannot move remote at '{0}', not identical".format(old)
raise raeting.StackError(emsg)
remote.uid = new
index = self.uidRemotes.keys().index(old)
del self.uidRemotes[old]
self.uidRemotes.insert(index, new, remote)
def renameRemote(self, remote, new):
'''
rename remote with old remote.name to new name but keep same index
'''
old = remote.name
if new != old:
if new in self.nameRemotes or new == self.local.name:
emsg = "Cannot rename remote to '{0}', already exists".format(new)
raise raeting.StackError(emsg)
if old not in self.nameRemotes:
emsg = "Cannot rename remote '{0}', does not exist".format(old)
raise raeting.StackError(emsg)
if remote is not self.nameRemotes[old]:
emsg = "Cannot rename remote '{0}', not identical".format(old)
raise raeting.StackError(emsg)
remote.name = new
index = self.nameRemotes.keys().index(old)
del self.nameRemotes[old]
self.nameRemotes.insert(index, new, remote)
def removeRemote(self, remote):
'''
Remove remote from all remotes dicts
'''
uid = remote.uid
if uid not in self.uidRemotes:
emsg = "Cannot remove remote '{0}', does not exist".format(uid)
raise raeting.StackError(emsg)
if remote is not self.uidRemotes[uid]:
emsg = "Cannot remove remote '{0}', not identical".format(uid)
raise raeting.StackError(emsg)
del self.uidRemotes[uid]
del self.nameRemotes[remote.name]
def removeAllRemotes(self):
'''
Remove all the remotes
'''
remotes = self.remotes.values() #make copy since changing .remotes in-place
for remote in remotes:
self.removeRemote(remote)
def fetchUidByName(self, name):
'''
Search for remote with matching name
Return remote if found Otherwise return None
'''
remote = self.nameRemotes.get(name)
return (remote.uid if remote else None)
def incStat(self, key, delta=1):
'''
Increment stat key counter by delta
'''
if key in self.stats:
self.stats[key] += delta
else:
self.stats[key] = delta
def updateStat(self, key, value):
'''
Set stat key to value
'''
self.stats[key] = value
def clearStat(self, key):
'''
Set the specified state counter to zero
'''
if key in self.stats:
self.stats[key] = 0
def clearStats(self):
'''
Set all the stat counters to zero and reset the timer
'''
for key, value in self.stats.items():
self.stats[key] = 0
self.statTimer.restart()
def _handleOneReceived(self):
'''
Handle one received message from server
assumes that there is a server
'''
try:
rx, ra = self.server.receive() # if no data the duple is ('',None)
except socket.error as ex:
err = raeting.get_exception_error(ex)
if err == errno.ECONNRESET:
return False
if not rx: # no received data
return False
self.rxes.append((rx, ra)) # duple = ( packet, source address)
return True
def serviceReceives(self):
'''
Retrieve from server all recieved and put on the rxes deque
'''
if self.server:
while self._handleOneReceived():
pass
def serviceReceiveOnce(self):
'''
Retrieve from server one recieved and put on the rxes deque
'''
if self.server:
self._handleOneReceived()
def _handleOneRx(self):
'''
Handle on message from .rxes deque
Assumes that there is a message on the .rxes deque
'''
raw, sa = self.rxes.popleft()
console.verbose("{0} received raw message\n{1}\n".format(self.name, raw))
processRx(packet=raw)
def serviceRxes(self):
'''
Process all messages in .rxes deque
'''
while self.rxes:
self._handleOneRx()
def serviceRxOnce(self):
'''
Process one messages in .rxes deque
'''
if self.rxes:
self._handleOneRx()
def processRx(self, packet):
'''
Process
'''
pass
def transmit(self, msg, uid=None):
'''
Append duple (msg, uid) to .txMsgs deque
If msg is not mapping then raises exception
If uid is None then it will default to the first entry in .remotes
'''
if not isinstance(msg, Mapping):
emsg = "Invalid msg, not a mapping {0}\n".format(msg)
console.terse(emsg)
self.incStat("invalid_transmit_body")
return
if uid is None:
if not self.remotes:
emsg = "No remote to send to\n"
console.terse(emsg)
self.incStat("invalid_destination")
return
uid = self.remotes.values()[0].uid
self.txMsgs.append((msg, uid))
def _handleOneTxMsg(self):
'''
Take one message from .txMsgs deque and handle it
Assumes there is a message on the deque
'''
body, uid = self.txMsgs.popleft() # duple (body dict, destination uid
self.message(body, uid=uid)
console.verbose("{0} sending\n{1}\n".format(self.name, body))
def serviceTxMsgs(self):
'''
Service .txMsgs queue of outgoing messages
'''
while self.txMsgs:
self._handleOneTxMsg()
def serviceTxMsgOnce(self):
'''
Service one message on .txMsgs queue of outgoing messages
'''
if self.txMsgs:
self._handleOneTxMsg()
def message(self, body, uid=None):
'''
Sends message body to remote at uid
'''
pass
def tx(self, packed, duid):
'''
Queue duple of (packed, da) on stack .txes queue
Where da is the ip destination (host,port) address associated with
the remote identified by duid
'''
if duid not in self.remotes:
msg = "Invalid destination remote id '{0}'".format(duid)
raise raeting.StackError(msg)
self.txes.append((packed, self.remotes[duid].ha))
def _handleOneTx(self, laters, blocks):
'''
Handle one message on .txes deque
Assumes there is a message
laters is deque of messages to try again later
blocks is list of destinations that already blocked on this service
'''
tx, ta = self.txes.popleft() # duple = (packet, destination address)
if ta in blocks: # already blocked on this iteration
laters.append((tx, ta)) # keep sequential
return
try:
self.server.send(tx, ta)
except socket.error as ex:
err = raeting.get_exception_error(ex)
errors = [errno.EAGAIN,
errno.EWOULDBLOCK,
errno.ENETUNREACH,
errno.EHOSTUNREACH,
errno.EHOSTDOWN,
errno.ECONNRESET]
if hasattr(errno, 'ETIME'):
errors.append[errno.ETIME]
if (err in errors):
# problem sending such as busy with last message. save it for later
laters.append((tx, ta))
blocks.append(ta)
else:
raise
def serviceTxes(self):
'''
Service the .txes deque to send messages through server
'''
if self.server:
laters = deque()
blocks = []
while self.txes:
self._handleOneTx(laters, blocks)
while laters:
self.txes.append(laters.popleft())
def serviceTxOnce(self):
'''
Service on message on the .txes deque to send through server
'''
if self.server:
laters = deque()
blocks = [] # will always be empty since only once
if self.txes:
self._handleOneTx(laters, blocks)
while laters:
self.txes.append(laters.popleft())
def serviceAllRx(self):
'''
Service:
server receive
rxes queue
process
'''
self.serviceReceives()
self.serviceRxes()
self.process()
def serviceAllTx(self):
'''
Service:
txMsgs queue
txes queue to server send
'''
self.serviceTxMsgs()
self.serviceTxes()
def serviceAll(self):
'''
Service or Process:
server receive
rxes queue
process
txMsgs queue
txes queue to server send
'''
self.serviceAllRx()
self.serviceAllTx()
def serviceServer(self):
'''
Service the server's receive and transmit queues
'''
self.serviceReceives()
self.serviceTxes()
def serviceOneAllRx(self):
'''
Propagate one packet all the way through the received side of the stack
Service:
server receive
rxes queue
process
'''
self.serviceReceiveOnce()
self.serviceRxOnce()
self.process()
def serviceOneAllTx(self):
'''
Propagate one packet all the way through the transmit side of the stack
Service:
txMsgs queue
txes queue to server send
'''
self.serviceTxMsgOnce()
self.serviceTxOnce()
def process(self):
'''
Allow timer based processing
'''
pass
def setUp(self):
self.store = storing.Store(stamp=0.0)
self.timer = StoreTimer(store=self.store, duration=1.0)
self.port = 4506
self.opts = dict(master_port=self.port)
def setUp(self):
self.store = Store(stamp=0.0)
self.timer = StoreTimer(store=self.store, duration=1.0)
self.dirpathBase = tempfile.mkdtemp(prefix="raet",
suffix="base",
dir=TEMPDIR)
stacking.RoadStack.Bk = raeting.BodyKind.json.value
#main stack
mainName = "main"
mainDirpath = os.path.join(self.dirpathBase, 'road', 'keep', mainName)
signer = nacling.Signer()
mainSignKeyHex = signer.keyhex
mainVerKeyHex = signer.verhex
privateer = nacling.Privateer()
mainPriKeyHex = privateer.keyhex
mainPubKeyHex = privateer.pubhex
#other stack
otherName = "other"
otherDirpath = os.path.join(self.dirpathBase, 'road', 'keep',
otherName)
signer = nacling.Signer()
otherSignKeyHex = signer.keyhex
otherVerKeyHex = signer.verhex
privateer = nacling.Privateer()
otherPriKeyHex = privateer.keyhex
otherPubKeyHex = privateer.pubhex
keeping.clearAllKeep(mainDirpath)
keeping.clearAllKeep(otherDirpath)
self.main = stacking.RoadStack(name=mainName,
uid=1,
sigkey=mainSignKeyHex,
prikey=mainPriKeyHex,
auto=raeting.AutoMode.once.value,
main=True,
dirpath=mainDirpath,
store=self.store)
remote1 = estating.RemoteEstate(
stack=self.main,
uid=2,
name=otherName,
ha=("127.0.0.1", raeting.RAET_TEST_PORT),
verkey=otherVerKeyHex,
pubkey=otherPubKeyHex,
)
self.main.addRemote(remote1)
self.other = stacking.RoadStack(name=otherName,
uid=2,
auto=raeting.AutoMode.once.value,
ha=("", raeting.RAET_TEST_PORT),
sigkey=otherSignKeyHex,
prikey=otherPriKeyHex,
dirpath=otherDirpath,
store=self.store)
remote0 = estating.RemoteEstate(
stack=self.other,
uid=3,
name=mainName,
ha=('127.0.0.1', raeting.RAET_PORT),
verkey=mainVerKeyHex,
pubkey=mainPubKeyHex,
)
self.other.addRemote(remote0)
remote0.publee = nacling.Publican(key=remote1.privee.pubhex)
remote1.publee = nacling.Publican(key=remote0.privee.pubhex)
stuff = []
for i in range(300):
stuff.append(str(i).rjust(4, " "))
self.stuff = ns2b("".join(stuff))
self.data = odict(hk=raeting.HeadKind.raet.value)
def test_get_backend():
"""
"""
print("Testing Falcon Example Backend Call")
store = Store(stamp=0.0) # create store
priming.setupTest()
valet = Valet(
port=8101,
bufsize=131072,
store=store,
app=exapp,
)
result = valet.open()
assert result
assert valet.servant.ha == ('0.0.0.0', 8101)
assert valet.servant.eha == ('127.0.0.1', 8101)
path = "http://{}:{}{}".format('localhost', valet.servant.eha[1],
"/example/backend")
headers = odict([('Accept', 'application/json'), ('Content-Length', 0)])
patron = Patron(
bufsize=131072,
store=store,
method='GET',
path=path,
headers=headers,
reconnectable=True,
)
assert patron.connector.reopen()
assert patron.connector.accepted == False
assert patron.connector.connected == False
assert patron.connector.cutoff == False
patron.transmit()
timer = StoreTimer(store, duration=1.0)
while (patron.requests or patron.connector.txes or not patron.responses
or not valet.idle()):
valet.serviceAll()
time.sleep(0.05)
patron.serviceAll()
time.sleep(0.05)
store.advanceStamp(0.1)
assert patron.connector.accepted == True
assert patron.connector.connected == True
assert patron.connector.cutoff == False
assert len(valet.servant.ixes) == 1
assert len(valet.reqs) == 1
assert len(valet.reps) == 1
requestant = valet.reqs.values()[0]
assert requestant.method == patron.requester.method
assert requestant.url == patron.requester.path
assert requestant.headers == {
'accept': 'application/json',
'accept-encoding': 'identity',
'content-length': '0',
'host': 'localhost:8101'
}
assert len(patron.responses) == 1
rep = patron.responses.popleft()
assert rep['status'] == 200
assert rep['reason'] == 'OK'
assert rep['body'] == bytearray(
b'{\n "approved": true,\n "body": "\\nHello World\\n\\n"\n}')
assert rep['data'] == odict([('approved', True),
('body', '\nHello World\n\n')])
responder = valet.reps.values()[0]
assert responder.status.startswith(str(rep['status']))
assert responder.headers == rep['headers']
# test for error by sending query arg path
#request = odict([('method', 'GET'),
#('path', '/example/backend'),
#('qargs', odict(path='/unknown')),
#('fragment', u''),
#('headers', odict([('Accept', 'application/json'),
#('Content-Length', 0)])),
#])
#patron.requests.append(request)
headers = odict([('Accept', 'application/json'), ('Content-Length', 0)])
patron.request(method='GET',
path='/example/backend',
qargs=odict(path='/unknown'),
headers=headers)
timer = StoreTimer(store, duration=1.0)
while (patron.requests or patron.connector.txes or not patron.responses
or not valet.idle()):
valet.serviceAll()
time.sleep(0.05)
patron.serviceAll()
time.sleep(0.05)
store.advanceStamp(0.1)
assert len(patron.responses) == 1
rep = patron.responses.popleft()
assert rep['status'] == 404
assert rep['reason'] == 'Not Found'
assert rep['body'] == bytearray(b'404 Not Found\nBackend Validation'
b' Error\nError backend validation.'
b' unknown\n')
assert not rep['data']
valet.close()
patron.close()
print("Done Test")
class BasicTestCase(unittest.TestCase):
""""""
def setUp(self):
self.store = Store(stamp=0.0)
self.timer = StoreTimer(store=self.store, duration=1.0)
if sys.platform == 'win32':
self.tempDirpath = tempfile.mktemp(prefix="raet", suffix="base",
dir=TEMPDIR)
else:
self.tempDirpath = tempfile.mkdtemp(prefix="raet", suffix="base",
dir=TEMPDIR)
self.baseDirpath = os.path.join(self.tempDirpath, 'lane', 'keep')
# main stack
self.main = stacking.LaneStack(name='main',
uid=1,
lanename='cherry',
sockdirpath=self.baseDirpath)
#other stack
self.other = stacking.LaneStack(name='other',
uid=1,
lanename='cherry',
sockdirpath=self.baseDirpath)
def tearDown(self):
self.main.server.close()
self.other.server.close()
if not sys.platform == 'win32':
shutil.rmtree(self.tempDirpath)
def service(self, duration=1.0, real=True):
'''
Utility method to service queues. Call from test method.
'''
self.timer.restart(duration=duration)
while not self.timer.expired:
self.other.serviceAll()
self.main.serviceAll()
self.store.advanceStamp(0.1)
if real:
time.sleep(0.1)
def serviceOther(self, duration=1.0, real=True):
'''
Utility method to only service other queues. Call from test method.
'''
self.timer.restart(duration=duration)
while not self.timer.expired:
self.other.serviceAll()
self.store.advanceStamp(0.1)
if real:
time.sleep(0.1)
def serviceMain(self, duration=1.0, real=True):
'''
Utility method to only service main queues. Call from test method.
'''
self.timer.restart(duration=duration)
while not self.timer.expired:
self.main.serviceAll()
self.store.advanceStamp(0.1)
if real:
time.sleep(0.1)
def bootstrap(self, kind=raeting.PackKind.json.value):
'''
Basic messaging
'''
self.main.addRemote(yarding.RemoteYard(stack=self.main, ha=self.other.ha))
self.other.addRemote(yarding.RemoteYard(stack=self.other, ha=self.main.ha))
self.assertEqual(self.main.name, 'main')
self.assertEqual(self.main.local.name, 'main')
self.assertEqual(self.main.ha, os.path.join(self.baseDirpath, 'cherry.main.uxd'))
self.assertEqual(len(self.main.remotes), 1)
remote = self.main.remotes.values()[0]
self.assertEqual(remote.ha, os.path.join(self.baseDirpath, 'cherry.other.uxd'))
self.assertEqual(remote.name, 'other')
self.assertTrue(remote.uid in self.main.remotes)
self.assertTrue(remote.name in self.main.nameRemotes)
self.assertTrue(remote.ha in self.main.haRemotes)
self.assertIs(self.main.nameRemotes[remote.name], remote)
self.assertIs(self.main.haRemotes[remote.ha], remote)
self.assertEqual(self.other.name, 'other')
self.assertEqual(self.other.local.name, 'other')
self.assertEqual(self.other.ha, os.path.join(self.baseDirpath, 'cherry.other.uxd'))
self.assertEqual(len(self.other.remotes), 1)
remote = self.other.remotes.values()[0]
self.assertEqual(remote.ha, os.path.join(self.baseDirpath, 'cherry.main.uxd'))
self.assertEqual(remote.name, 'main')
self.assertTrue(remote.uid in self.other.remotes)
self.assertTrue(remote.name in self.other.nameRemotes)
self.assertTrue(remote.ha in self.other.haRemotes)
self.assertIs(self.other.nameRemotes[remote.name], remote)
self.assertIs(self.other.haRemotes[remote.ha], remote)
stacking.LaneStack.Pk = kind
def message(self, mains, others, duration=1.0):
'''
Transmit and receive messages in mains and others lists
'''
for msg in mains:
self.main.transmit(msg=msg)
for msg in others:
self.other.transmit(msg=msg)
self.service(duration=duration)
self.assertEqual(len(self.main.rxMsgs), len(others))
for i, duple in enumerate(self.main.rxMsgs):
console.terse("Yard '{0}' rxed:\n'{1}'\n".format(self.main.local.name, duple))
self.assertDictEqual(others[i], duple[0])
self.assertEqual(len(self.other.rxMsgs), len(mains))
for i, duple in enumerate(self.other.rxMsgs):
console.terse("Yard '{0}' rxed:\n'{1}'\n".format(self.other.local.name, duple))
self.assertDictEqual(mains[i], duple[0])
def createLaneData(self, name, uid, base, lanename):
'''
Creates odict and populates with data to setup lane stack
{
name: stack name
dirpath: dirpath for keep files
lanename: name of yard
}
'''
data = odict()
data['name'] = name
data['uid'] = uid
data['sockdirpath'] = os.path.join(base, name)
data['lanename'] = lanename
return data
def createLaneStack(self, data, main=None):
'''
Creates stack and local yard from data
returns stack
'''
stack = stacking.LaneStack(name=data['name'],
uid=data['uid'],
main=main,
lanename=data['lanename'],
sockdirpath=data['sockdirpath'])
return stack
def serviceMainOther(self, main, other, duration=1.0):
'''
Utility method to service queues. Call from test method.
'''
self.timer.restart(duration=duration)
while not self.timer.expired:
other.serviceAll()
main.serviceAll()
self.store.advanceStamp(0.1)
time.sleep(0.1)
def messageMainOther(self, main, other, mains, others, duration=1.0):
'''
Utility to send messages both ways
'''
for msg in mains:
main.transmit(msg, uid=main.fetchUidByName(other.local.name))
for msg in others:
other.transmit(msg, uid=other.fetchUidByName(main.local.name))
self.serviceMainOther(main, other, duration=duration)
self.assertEqual(len(main.rxMsgs), len(others))
for i, duple in enumerate(main.rxMsgs):
console.terse("Yard '{0}' rxed:\n'{1}'\n".format(main.local.name, duple))
self.assertDictEqual(others[i], duple[0])
self.assertEqual(len(other.rxMsgs), len(mains))
for i, duple in enumerate(other.rxMsgs):
console.terse("Yard '{0}' rxed:\n'{1}'\n".format(other.local.name, duple))
self.assertDictEqual(mains[i], duple[0])
def serviceStackOneTx(self, stack):
'''
Utility method to service one packet on Tx queues. Call from test method.
'''
stack.serviceOneAllTx()
time.sleep(0.1)
self.store.advanceStamp(0.1)
def serviceStackOneRx(self, stack):
'''
Utility method to service one packet on Rx queues. Call from test method.
'''
stack.serviceOneAllRx()
time.sleep(0.1)
self.store.advanceStamp(0.1)
def serviceOneTx(self, main, other):
'''
Utility method to service one packet on Tx queues. Call from test method.
'''
other.serviceOneAllTx()
main.serviceOneAllTx()
time.sleep(0.1)
self.store.advanceStamp(0.1)
def serviceOneRx(self, main, other):
'''
Utility method to service one packet on Rx queues. Call from test method.
'''
other.serviceOneAllRx()
main.serviceOneAllRx()
time.sleep(0.1)
self.store.advanceStamp(0.1)
def serviceOneAll(self, main, other):
'''
Utility method to service one packet on all queues. Call from test method.
'''
self.serviceOneTx(main=main, other=other)
self.serviceOneRx(main=main, other=other)
def serviceStack(self, stack, duration=1.0):
'''
Utility method to service queues for one stack. Call from test method.
'''
self.timer.restart(duration=duration)
while not self.timer.expired:
stack.serviceAll()
self.store.advanceStamp(0.1)
time.sleep(0.1)
def serviceStacks(self, stacks, duration=1.0):
'''
Utility method to service queues for list of stacks. Call from test method.
'''
self.timer.restart(duration=duration)
while not self.timer.expired:
for stack in stacks:
stack.serviceAll()
self.store.advanceStamp(0.1)
time.sleep(0.1)
def testMessageJson(self):
'''
Basic messaging with json packing
'''
console.terse("{0}\n".format(self.testMessageJson.__doc__))
self.bootstrap(kind=raeting.PackKind.json.value)
mains = []
mains.append(odict(what="This is a message to the serf. Get to Work", extra="Fix the fence."))
others = []
others.append(odict(what="This is a message to the lord. Let me be", extra="Go away."))
self.message(mains=mains, others=others)
def testMessageMsgpack(self):
'''
Basic messaging with msgpack packing
'''
console.terse("{0}\n".format(self.testMessageMsgpack.__doc__))
self.bootstrap(kind=raeting.PackKind.pack.value)
mains = []
mains.append(odict(what="This is a message to the serf. Get to Work", extra="Fix the fence."))
others = []
others.append(odict(what="This is a message to the lord. Let me be", extra="Go away."))
self.message(mains=mains, others=others)
def testMessageMultipleJson(self):
'''
Multiple messages with json packing
'''
console.terse("{0}\n".format(self.testMessageMultipleJson.__doc__))
self.bootstrap(kind=raeting.PackKind.json.value)
mains = []
mains.append(odict([('house', "Mama mia1"), ('queue', "fix me")]))
mains.append(odict([('house', "Mama mia2"), ('queue', "stop me")]))
mains.append(odict([('house', "Mama mia3"), ('queue', "help me")]))
mains.append(odict([('house', "Mama mia4"), ('queue', "run me")]))
others = []
others.append(odict([('house', "Papa pia1"), ('queue', "fix me")]))
others.append(odict([('house', "Papa pia1"), ('queue', "stop me")]))
others.append(odict([('house', "Papa pia1"), ('queue', "help me")]))
others.append(odict([('house', "Papa pia1"), ('queue', "run me")]))
self.message(mains=mains, others=others)
def testMessageMultipleMsgpack(self):
'''
multiple messages with msgpack packing
'''
console.terse("{0}\n".format(self.testMessageMultipleMsgpack.__doc__))
self.bootstrap(kind=raeting.PackKind.pack.value)
mains = []
mains.append(odict([('house', "Mama mia1"), ('queue', "fix me")]))
mains.append(odict([('house', "Mama mia2"), ('queue', "stop me")]))
mains.append(odict([('house', "Mama mia3"), ('queue', "help me")]))
mains.append(odict([('house', "Mama mia4"), ('queue', "run me")]))
others = []
others.append(odict([('house', "Papa pia1"), ('queue', "fix me")]))
others.append(odict([('house', "Papa pia1"), ('queue', "stop me")]))
others.append(odict([('house', "Papa pia1"), ('queue', "help me")]))
others.append(odict([('house', "Papa pia1"), ('queue', "run me")]))
self.message(mains=mains, others=others)
def testMessageSectionedJson(self):
'''
Sectioned messages with json packing
'''
console.terse("{0}\n".format(self.testMessageSectionedJson.__doc__))
self.bootstrap(kind=raeting.PackKind.json.value)
#big packets
stuff = []
for i in range(10000):
stuff.append(str(i).rjust(10, " "))
stuff = "".join(stuff)
src = ['mayor', self.main.local.name, None]
dst = ['citizen', self.other.local.name, None]
route = odict([('src', src), ('dst', dst)])
mains = []
mains.append(odict([('route', route), ('content', stuff)]))
src = ['citizen', self.other.local.name, None]
dst = ['mayor', self.main.local.name, None]
route = odict([('src', src), ('dst', dst)])
others = []
others.append(odict([('route', route), ('content', stuff)]))
self.message(mains=mains, others=others, duration=2.0)
def testMessageSectionedMsgpack(self):
'''
Sectioned messages with msgpack packing
'''
console.terse("{0}\n".format(self.testMessageSectionedMsgpack.__doc__))
self.bootstrap(kind=raeting.PackKind.pack.value)
#big packets
stuff = []
for i in range(10000):
stuff.append(str(i).rjust(10, " "))
stuff = "".join(stuff)
src = ['mayor', self.main.local.name, None]
dst = ['citizen', self.other.local.name, None]
route = odict([('src', src), ('dst', dst)])
mains = []
mains.append(odict([('route', route), ('content', stuff)]))
src = ['citizen', self.other.local.name, None]
dst = ['mayor', self.main.local.name, None]
route = odict([('src', src), ('dst', dst)])
others = []
others.append(odict([('route', route), ('content', stuff)]))
self.message(mains=mains, others=others, duration=2.0)
def testAutoAccept(self):
'''
Basic send auto accept message
'''
console.terse("{0}\n".format(self.testAutoAccept.__doc__))
self.assertTrue(self.main.accept)
# Don't add remote yard to main so only way to get message from other is
# if auto acccept works
self.other.addRemote(yarding.RemoteYard(stack=self.other, ha=self.main.ha))
self.assertEqual(self.main.name, 'main')
self.assertEqual(self.main.local.name, 'main')
self.assertEqual(self.main.ha, os.path.join(self.baseDirpath, 'cherry.main.uxd'))
self.assertEqual(len(self.main.remotes), 0)
self.assertEqual(self.other.name, 'other')
self.assertEqual(self.other.local.name, 'other')
self.assertEqual(self.other.ha, os.path.join(self.baseDirpath, 'cherry.other.uxd'))
self.assertEqual(len(self.other.remotes), 1)
remote = self.other.remotes.values()[0]
self.assertEqual(remote.ha, os.path.join(self.baseDirpath, 'cherry.main.uxd'))
self.assertEqual(remote.name, 'main')
self.assertTrue(remote.uid in self.other.remotes)
self.assertTrue(remote.name in self.other.nameRemotes)
self.assertTrue(remote.ha in self.other.haRemotes)
self.assertIs(self.other.nameRemotes[remote.name], remote)
self.assertIs(self.other.haRemotes[remote.ha], remote)
stacking.LaneStack.Pk = raeting.PackKind.pack.value
others = []
others.append(odict(what="This is a message to the lord. Let me be", extra="Go away."))
self.message(mains=[], others=others)
self.assertEqual(len(self.main.remotes), 1)
remote = self.main.remotes.values()[0]
self.assertEqual(remote.ha, os.path.join(self.baseDirpath, 'cherry.other.uxd'))
self.assertEqual(remote.name, 'other')
self.assertTrue(remote.uid in self.main.remotes)
self.assertTrue(remote.name in self.main.nameRemotes)
self.assertTrue(remote.ha in self.main.haRemotes)
self.assertIs(self.main.nameRemotes[remote.name], remote)
self.assertIs(self.main.haRemotes[remote.ha], remote)
self.main.rxMsgs = deque()
self.other.rxMsgs = deque()
mains = []
mains.append(odict(what="This is a message to the serf. Get to Work", extra="Fix the fence."))
self.message(mains=mains, others=[])
def testAutoAcceptNot(self):
'''
Basic send non auto accept message
'''
console.terse("{0}\n".format(self.testAutoAcceptNot.__doc__))
self.main.accept = False
self.assertIs(self.main.accept, False)
# Don't add remote yard to main so only way to get message from other is
# if auto acccept works
self.other.addRemote(yarding.RemoteYard(stack=self.other, ha=self.main.ha))
self.assertEqual(self.main.name, 'main')
self.assertEqual(self.main.local.name, 'main')
self.assertEqual(self.main.ha, os.path.join(self.baseDirpath, 'cherry.main.uxd'))
self.assertEqual(len(self.main.remotes), 0)
self.assertEqual(self.other.name, 'other')
self.assertEqual(self.other.local.name, 'other')
self.assertEqual(self.other.ha, os.path.join(self.baseDirpath, 'cherry.other.uxd'))
self.assertEqual(len(self.other.remotes), 1)
remote = self.other.remotes.values()[0]
self.assertEqual(remote.ha, os.path.join(self.baseDirpath, 'cherry.main.uxd'))
self.assertEqual(remote.name, 'main')
self.assertTrue(remote.uid in self.other.remotes)
self.assertTrue(remote.name in self.other.nameRemotes)
self.assertTrue(remote.ha in self.other.haRemotes)
self.assertIs(self.other.nameRemotes[remote.name], remote)
self.assertIs(self.other.haRemotes[remote.ha], remote)
stacking.LaneStack.Pk = raeting.PackKind.pack.value
others = []
others.append(odict(what="This is a message to the lord. Let me be", extra="Go away."))
for msg in others:
self.other.transmit(msg=msg)
self.service()
self.assertEqual(len(self.main.rxMsgs), 0)
self.assertEqual(len(self.main.remotes), 0)
self.assertEqual(self.main.stats['unaccepted_source_yard'], 1)
def testFetchRemoteFromHa(self):
'''
Fetching remote yard by HA
'''
console.terse("{0}\n".format(self.testFetchRemoteFromHa.__doc__))
self.bootstrap(kind=raeting.PackKind.json.value)
for remote in self.main.remotes.values():
fetched = self.main.haRemotes.get(remote.ha)
self.assertIs(remote, fetched)
for remote in self.other.remotes.values():
fetched = self.other.haRemotes.get(remote.ha)
self.assertIs(remote, fetched)
def testRestart(self):
'''
Test messaging after restart
'''
console.terse("{0}\n".format(self.testRestart.__doc__))
stacking.LaneStack.Pk = raeting.PackKind.json.value
mainData = self.createLaneData(name='main',
uid=1,
base=self.baseDirpath,
lanename='apple')
main = self.createLaneStack(data=mainData, main=True)
self.assertTrue(main.ha.endswith(os.path.join('lane','keep','main',
'apple.main.uxd')))
self.assertTrue(main.main)
otherData = self.createLaneData(name='other',
uid=1,
base=self.baseDirpath,
lanename='apple')
other = self.createLaneStack(data=otherData)
self.assertTrue(other.ha.endswith(os.path.join('lane','keep','other',
'apple.other.uxd')))
main.addRemote(yarding.RemoteYard(stack=main, ha=other.ha))
self.assertTrue('other' in main.nameRemotes)
self.assertTrue(other.ha in main.haRemotes)
other.addRemote(yarding.RemoteYard(stack=other, ha=main.ha))
self.assertTrue('main' in other.nameRemotes)
self.assertTrue(main.ha in other.haRemotes)
src = ['mayor', main.local.name, None] # (house, yard, queue)
dst = ['citizen', other.local.name, None]
route = odict([('src', src), ('dst', dst)])
stuff = "This is my command"
mains = []
mains.append(odict([('route', route), ('content', stuff)]))
src = ['citizen', other.local.name, None]
dst = ['mayor', main.local.name, None]
route = odict([('src', src), ('dst', dst)])
stuff = "This is my reply."
others = []
others.append(odict([('route', route), ('content', stuff)]))
self.messageMainOther(main, other, mains, others, duration=1.0)
self.assertEqual(len(main.remotes), 1)
self.assertTrue('other' in main.nameRemotes)
self.assertEqual(len(other.remotes), 1)
self.assertTrue('main' in other.nameRemotes)
self.assertNotEqual(main.nameRemotes['other'].sid, 0)
self.assertNotEqual(other.nameRemotes['main'].sid, 0)
self.assertEqual(main.nameRemotes['other'].rsid,
other.nameRemotes['main'].sid)
self.assertEqual(other.nameRemotes['main'].rsid,
main.nameRemotes['other'].sid)
#now close down make new stacks
main.server.close()
other.server.close()
main = self.createLaneStack(data=mainData, main=True)
other = self.createLaneStack(data=otherData)
main.addRemote(yarding.RemoteYard(stack=main, ha=other.ha))
self.assertTrue('other' in main.nameRemotes)
other.addRemote(yarding.RemoteYard(stack=other, ha=main.ha))
self.assertTrue('main' in other.nameRemotes)
self.assertEqual(len(main.remotes), 1)
self.assertEqual(len(other.remotes), 1)
self.assertNotEqual(main.nameRemotes['other'].sid, 0)
self.assertNotEqual(other.nameRemotes['main'].sid, 0)
self.assertEqual(main.nameRemotes['other'].rsid, 0)
self.assertEqual(other.nameRemotes['main'].rsid, 0)
self.messageMainOther(main, other, mains, others, duration=1.0)
self.assertEqual(main.nameRemotes['other'].rsid,
other.nameRemotes['main'].sid)
self.assertEqual(other.nameRemotes['main'].rsid,
main.nameRemotes['other'].sid)
#now close down make new stacks
main.server.close()
other.server.close()
main = self.createLaneStack(data=mainData, main=True)
other = self.createLaneStack(data=otherData)
main.addRemote(yarding.RemoteYard(stack=main, ha=other.ha))
self.assertTrue('other' in main.nameRemotes)
other.addRemote(yarding.RemoteYard(stack=other, ha=main.ha))
self.assertTrue('main' in other.nameRemotes)
self.assertEqual(len(main.remotes), 1)
self.assertEqual(len(other.remotes), 1)
self.assertNotEqual(main.nameRemotes['other'].sid, 0)
self.assertNotEqual(other.nameRemotes['main'].sid, 0)
self.assertEqual(main.nameRemotes['other'].rsid, 0)
self.assertEqual(other.nameRemotes['main'].rsid, 0)
# now send paginated messages
src = ['mayor', main.local.name, None] # (house, yard, queue)
dst = ['citizen', other.local.name, None]
route = odict([('src', src), ('dst', dst)])
stuff = ["Do as I say."]
for i in range(10000):
stuff.append(str(i).rjust(10, " "))
stuff = "".join(stuff)
mains = []
mains.append(odict([('route', route), ('content', stuff)]))
src = ['citizen', other.local.name, None]
dst = ['mayor', main.local.name, None]
route = odict([('src', src), ('dst', dst)])
stuff = ["As you wish."]
for i in range(10000):
stuff.append(str(i).rjust(10, " "))
stuff = "".join(stuff)
others = []
others.append(odict([('route', route), ('content', stuff)]))
self.messageMainOther(main, other, mains, others, duration=1.0)
self.assertEqual(main.nameRemotes['other'].rsid,
other.nameRemotes['main'].sid)
self.assertEqual(other.nameRemotes['main'].rsid,
main.nameRemotes['other'].sid)
#now close down make new stacks send page at a time
main.server.close()
other.server.close()
main = self.createLaneStack(data=mainData, main=True)
other = self.createLaneStack(data=otherData)
main.addRemote(yarding.RemoteYard(stack=main, ha=other.ha))
self.assertTrue('other' in main.nameRemotes)
other.addRemote(yarding.RemoteYard(stack=other, ha=main.ha))
self.assertTrue('main' in other.nameRemotes)
self.assertEqual(len(main.remotes), 1)
self.assertEqual(len(other.remotes), 1)
self.assertNotEqual(main.nameRemotes['other'].sid, 0)
self.assertNotEqual(other.nameRemotes['main'].sid, 0)
self.assertEqual(main.nameRemotes['other'].rsid, 0)
self.assertEqual(other.nameRemotes['main'].rsid, 0)
for msg in mains:
main.transmit(msg, uid=main.fetchUidByName(other.local.name))
for msg in others:
other.transmit(msg, uid=other.fetchUidByName(main.local.name))
self.assertEqual(len(main.txMsgs), 1)
self.assertEqual(len(other.txMsgs), 1)
self.assertEqual(len(main.nameRemotes['other'].books), 0)
self.assertEqual(len(other.nameRemotes['main'].books), 0)
self.assertEqual(len(main.rxMsgs), 0)
self.assertEqual(len(other.rxMsgs), 0)
# Now only send and receive one page to/from each side
self.serviceOneAll(main, other)
self.assertEqual(len(main.txMsgs), 0)
self.assertEqual(len(other.txMsgs), 0)
self.assertEqual(len(main.txes), 1)
self.assertEqual(len(other.txes), 1)
self.assertEqual(len(main.nameRemotes['other'].books), 1)
self.assertEqual(len(other.nameRemotes['main'].books), 1)
self.assertEqual(len(main.rxMsgs), 0)
self.assertEqual(len(other.rxMsgs), 0)
self.assertEqual(main.nameRemotes['other'].rsid,
other.nameRemotes['main'].sid)
self.assertEqual(other.nameRemotes['main'].rsid,
main.nameRemotes['other'].sid)
# save sids
mainSid = main.nameRemotes['other'].sid
otherSid = other.nameRemotes['main'].sid
#now close down one side only, make new stack
main.server.close()
main = self.createLaneStack(data=mainData, main=True)
main.addRemote(yarding.RemoteYard(stack=main, ha=other.ha))
self.assertEqual(len(main.remotes), 1)
self.assertEqual(len(other.remotes), 1)
self.assertNotEqual(main.nameRemotes['other'].sid, mainSid)
self.assertEqual(other.nameRemotes['main'].sid, otherSid)
self.assertEqual(main.nameRemotes['other'].rsid, 0)
self.assertEqual(other.nameRemotes['main'].rsid, mainSid)
self.assertEqual(len(main.txes), 0)
self.assertEqual(len(other.txes), 1)
self.assertEqual(len(main.nameRemotes['other'].books), 0)
self.assertEqual(len(other.nameRemotes['main'].books), 1)
self.assertEqual(len(main.rxMsgs), 0)
self.assertEqual(len(other.rxMsgs), 0)
# Now remaining page from other (there should be no pages from main)
self.serviceOneAll(main, other)
self.assertEqual(main.nameRemotes['other'].rsid,
other.nameRemotes['main'].sid)
self.assertNotEqual(other.nameRemotes['main'].rsid,
main.nameRemotes['other'].sid)
self.assertEqual(len(main.txes), 0)
self.assertEqual(len(other.txes), 0)
self.assertEqual(len(main.nameRemotes['other'].books), 0)
self.assertEqual(len(other.nameRemotes['main'].books), 1)
self.assertEqual(len(main.rxMsgs), 0)
self.assertEqual(len(other.rxMsgs), 0)
self.assertEqual(main.stats['missed_page'], 1)
#send a new message from main and reap stale book from other
for msg in mains:
main.transmit(msg, uid=main.fetchUidByName(other.local.name))
self.serviceMainOther(main, other, duration=1.0)
self.assertEqual(main.nameRemotes['other'].rsid,
other.nameRemotes['main'].sid)
self.assertEqual(other.nameRemotes['main'].rsid,
main.nameRemotes['other'].sid)
self.assertEqual(len(main.txes), 0)
self.assertEqual(len(other.txes), 0)
self.assertEqual(len(main.nameRemotes['other'].books), 0)
self.assertEqual(len(other.nameRemotes['main'].books), 0)
self.assertEqual(len(main.rxMsgs), 0)
self.assertEqual(len(main.rxMsgs), 0)
self.assertEqual(len(other.rxMsgs), 1)
self.assertEqual(other.stats['stale_book'], 1)
self.assertEqual(len(other.rxMsgs), len(mains))
for i, duple in enumerate(other.rxMsgs):
console.terse("Yard '{0}' rxed:\n'{1}'\n".format(other.local.name, duple))
self.assertDictEqual(mains[i], duple[0])
# setup to test reset sid numbering by sending single pages to create stale books
other.rxMsgs.pop()
for msg in mains:
main.transmit(msg, uid=main.fetchUidByName(other.local.name))
for msg in others:
other.transmit(msg, uid=other.fetchUidByName(main.local.name))
self.serviceOneAll(main, other)
self.assertEqual(main.nameRemotes['other'].rsid,
other.nameRemotes['main'].sid)
self.assertEqual(other.nameRemotes['main'].rsid,
main.nameRemotes['other'].sid)
self.assertEqual(len(main.txes), 1)
self.assertEqual(len(other.txes), 1)
self.assertEqual(len(main.nameRemotes['other'].books), 1)
self.assertEqual(len(other.nameRemotes['main'].books), 1)
self.assertEqual(len(main.rxMsgs), 0)
self.assertEqual(len(other.rxMsgs), 0)
# simulate restart that loses msg in queue
main.txes.pop()
other.txes.pop()
src = ['mayor', main.local.name, None] # (house, yard, queue)
dst = ['citizen', other.local.name, None]
route = odict([('src', src), ('dst', dst)])
stuff = "This is my command"
mains = []
mains.append(odict([('route', route), ('content', stuff)]))
src = ['citizen', other.local.name, None]
dst = ['mayor', main.local.name, None]
route = odict([('src', src), ('dst', dst)])
stuff = "This is my reply."
others = []
others.append(odict([('route', route), ('content', stuff)]))
mainSid = main.local.nextSid()
otherSid = other.local.nextSid()
main.nameRemotes['other'].sid = mainSid
other.nameRemotes['main'].sid = otherSid
for msg in mains:
main.transmit(msg, uid=main.fetchUidByName(other.local.name))
for msg in others:
other.transmit(msg, uid=other.fetchUidByName(main.local.name))
self.serviceOneAll(main, other)
self.assertEqual(main.nameRemotes['other'].sid, mainSid)
self.assertEqual(other.nameRemotes['main'].sid, otherSid)
self.assertEqual(main.nameRemotes['other'].rsid, otherSid)
self.assertEqual(other.nameRemotes['main'].rsid, mainSid)
self.assertEqual(len(main.txes), 0)
self.assertEqual(len(other.txes), 0)
self.assertEqual(len(main.nameRemotes['other'].books), 0)
self.assertEqual(len(other.nameRemotes['main'].books), 0)
self.assertEqual(len(main.rxMsgs), 1)
self.assertEqual(len(other.rxMsgs), 1)
self.assertEqual(main.stats['stale_book'], 1)
self.assertEqual(other.stats['stale_book'], 2)
main.server.close()
other.server.close()
class BasicLoadTestCase(unittest.TestCase):
'''
Base class for load tests. Provides generic master/slave runners with flooding and checking logic.
'''
def __init__(self, *args, **kwargs):
super(BasicLoadTestCase, self).__init__(*args, **kwargs)
def setUp(self):
self.store = Store(stamp=0.0)
self.timer = StoreTimer(store=self.store, duration=1.0)
self.baseDirpath = tempfile.mkdtemp(prefix="raet", suffix="base", dir=TEMPDIR)
self.stack = None
# This has to be set to True in the only one process that would perform tearDown steps
self.engine = True
stacking.RoadStack.BurstSize = 100
def tearDown(self):
# The only engine have to tear down stacks after all workers have done
if not self.engine:
if self.stack:
self.stack.server.close()
self.stack.clearAllDir()
else:
if os.path.exists(self.baseDirpath):
shutil.rmtree(self.baseDirpath)
def createStack(self, name, port):
'''
Create a RoadStack object bound to the specified port on localhost.
:param name: stack name
:param port: port to bind to
:return: the stack
'''
dirpath = os.path.join(self.baseDirpath, 'road', 'keep', name)
signer = nacling.Signer()
mainSignKeyHex = signer.keyhex
privateer = nacling.Privateer()
mainPriKeyHex = privateer.keyhex
keeping.clearAllKeep(dirpath)
return stacking.RoadStack(store=self.store,
name=name,
main=True,
auto=raeting.AutoMode.once.value,
ha=("", port),
sigkey=mainSignKeyHex,
prikey=mainPriKeyHex,
dirpath=dirpath,
)
def joinAll(self, initiator, correspondentAddresses, timeout=None, retryCount=1):
'''
Utility method to do join.
:param initiator: join initiator stack
:param correspondentAddresses: an iterable object containing ha tuples
:param timeout: join timeout, will be passed to stack
:param retryCount: retry join max the specified count of times until success
'''
console.terse("\nJoin Transaction **************\n")
addresses = list(correspondentAddresses)
for ha in addresses:
initiator.addRemote(estating.RemoteEstate(stack=initiator,
fuid=0, # vacuous join
sid=0, # always 0 for join
ha=ha))
while retryCount > 0:
done = True
for remote in initiator.remotes.values():
if not remote.joined:
done = False
initiator.join(uid=remote.uid, timeout=timeout)
self.serviceOne(initiator)
if done:
break
retryCount -= 1
def allowAll(self, initiator, timeout=None, retryCount=1):
'''
Utility method to do allow.
:param initiator: allow initiator stack
:param timeout: allow timeout, will be passed to stack
:param retryCount: retry allow max the specified count of times until success
'''
console.terse("\nAllow Transaction **************\n")
while retryCount > 0:
done = True
for remote in initiator.remotes.values():
if not remote.allowed:
done = False
initiator.allow(uid=remote.uid, timeout=timeout)
self.serviceOne(initiator)
if done:
break
retryCount -= 1
def serviceAll(self, stacks, duration=100.0, timeout=0.0, step=0.1, exitCase=None):
'''
Utility method to service queues.
:param stacks: iterable containing stacks to service
:param duration: max service duration, actually could finish earlier if there still no transactions to service
:param timeout: time to continue service even if there are no transactions
:param step: timer step
:param exitCase: stop immediately if this function return true
:return: actual exit reason, could be one of 'duration', 'exitCase' or 'timeout'
'''
self.timer.restart(duration=duration)
empty = False
elapsed = self.timer.getElapsed()
retReason = 'duration'
while not self.timer.expired:
for stack in stacks:
stack.serviceAll()
if any(stack.transactions for stack in stacks):
empty = False # reset nop timeout
else:
if exitCase and exitCase():
retReason = 'exitCase'
break
if empty:
if self.timer.getElapsed() - elapsed > timeout:
retReason = 'timeout'
break
else:
empty = True
elapsed = self.timer.getElapsed()
self.store.advanceStamp(step)
time.sleep(step)
return retReason
def serviceOne(self, stack, duration=100.0, timeout=0.0, step=0.1, exitCase=None):
'''
Utility method to service one stack. See :func:`serviceStacks`.
'''
return self.serviceAll([stack], duration, timeout, step, exitCase)
def bidirectional(self, stack, duration=3.0):
'''
Simultaneously sends and receives test messages.
- self.msgSize: message size
- self.msgCount: per remote message count
:param stack: the stack to service
:param duration: service duration to prevent hang
'''
console.terse("\nMessages Bidirectional {0} *********\n".format(stack.name))
verifier = data.MessageVerifier(size=self.msgSize, msgCount=self.msgCount, remoteCount=len(stack.remotes),
house='manor', queue='stuff')
received = 0
expected = len(stack.remotes) * self.msgCount
maxTransactions = MAX_TRANSACTIONS * len(stack.remotes)
for msg in data.generateMessages(name=stack.name, size=self.msgSize, count=self.msgCount):
for remote in stack.remotes.values():
# send message
stack.transmit(msg, uid=remote.uid)
# service while there are too much transactions (but loops at least once)
while True:
self.serviceOne(stack, duration=0.01, step=0.01)
# check received
if stack.rxMsgs:
received += len(stack.rxMsgs)
while stack.rxMsgs:
verifier.verifyMessage(stack.rxMsgs.popleft())
# keep servicing if there are a lot of transactions
if len(stack.transactions) <= maxTransactions:
break
# all sent, continue handle received
while received < expected:
self.serviceOne(stack, duration=duration, timeout=duration,
exitCase=lambda: stack.rxMsgs)
# if received nothing during timeout, assume we're done
if not stack.rxMsgs:
break
received += len(stack.rxMsgs)
while stack.rxMsgs:
verifier.verifyMessage(stack.rxMsgs.popleft())
# wait remaining messenger transactions if any to be closed
if stack.transactions:
self.serviceOne(stack, duration=duration)
# Done. Wait others
self.doneCounter.inc()
elapsed = 0.0 # be safe from engine unexpected stop
step = 1.0
while not self.stopFlag.get() and elapsed < duration * 2:
self.serviceOne(stack, duration=step, timeout=step)
elapsed += step
# check result
console.terse("\nStack '{0}' uid={1}\n\tTransactions: {2}\n\trcv/exp: {3}/{4}\n\tStats: {5}\n"
.format(stack.name, stack.local.uid, stack.transactions, received, expected, stack.stats))
rcvErrors = verifier.checkAllDone()
if rcvErrors:
console.terse("{0} received message with the following errors:\n".format(stack.name))
for s in rcvErrors:
console.terse("\t{0} from {1}\n".format(stack.name, s))
self.assertEqual(len(stack.transactions), 0)
self.assertEqual(len(rcvErrors), 0)
self.assertEqual(received, expected)
def send(self, stack, duration=3.0):
'''
Sends test messages.
- self.msgSize: message size
- self.msgCount: per remote message count
:param stack: the stack to service
:param duration: service duration to prevent hang
'''
console.terse("\nMessages Sender {0} *********\n".format(stack.name))
maxTransactions = MAX_TRANSACTIONS * len(stack.remotes)
for msg in data.generateMessages(name=stack.name, size=self.msgSize, count=self.msgCount):
for remote in stack.remotes.values():
# send message
stack.transmit(msg, uid=remote.uid)
# service while there are too much transactions (but loops at least once)
while True:
self.serviceOne(stack, duration=0.01, step=0.01)
if len(stack.transactions) <= maxTransactions:
break
# done, wait others
self.doneCounter.inc()
elapsed = 0.0 # be safe from engine unexpected stop
step = 1.0
while not self.stopFlag.get() and elapsed < duration * 2:
self.serviceOne(stack, duration=step, timeout=step)
elapsed += step
# check result
console.terse("\nStack '{0}' uid={1}\n\tTransactions: {2}\n\tStats: {3}\n"
.format(stack.name, stack.local.uid, stack.transactions, stack.stats))
self.assertEqual(len(stack.transactions), 0)
self.assertEqual(len(stack.rxMsgs), 0)
def receive(self, stack, duration=3.0):
'''
Receives test messages.
- self.msgSize: message size
- self.msgCount: per remote message count
:param stack: the stack to service
:param duration: service duration to prevent hang
'''
console.terse("\nMessages Receiver {0} *********\n".format(stack.name))
verifier = data.MessageVerifier(size=self.msgSize, msgCount=self.msgCount, remoteCount=len(stack.remotes),
house='manor', queue='stuff')
# Receive messages
received = 0
expected = len(stack.remotes) * self.msgCount
while received < expected:
reason = self.serviceOne(stack, duration=duration, timeout=duration,
exitCase=lambda: stack.rxMsgs)
console.terse("{0} service exit reason: {1}, transactions: {2}\n".format(
stack.name, reason, stack.transactions))
# received nothing during timeout, assume there is nothing to receive
if not stack.rxMsgs:
break
received += len(stack.rxMsgs)
while stack.rxMsgs:
verifier.verifyMessage(stack.rxMsgs.popleft())
# Done. Wait others
self.doneCounter.inc()
elapsed = 0.0 # be safe from engine unexpected stop
step = 1.0
while not self.stopFlag.get() and elapsed < duration * 2:
self.serviceOne(stack, duration=step, timeout=step)
elapsed += step
# Verify results
console.terse("\nStack '{0}' uid={1}\n\tTransactions: {2}\n\trcv/exp: {3}/{4}\n\tStats: {5}\n"
.format(stack.name, stack.local.uid, stack.transactions, received, expected, stack.stats))
for t in stack.transactions:
if isinstance(t, transacting.Messengent):
print("Tray lost segments: {0}\n".format([i for i, x in enumerate(t.tray.segments) if x is None]))
rcvErrors = verifier.checkAllDone()
if rcvErrors:
console.terse("{0} received message with the following errors:\n".format(stack.name))
for s in rcvErrors:
console.terse("\t{0} from {1}\n".format(stack.name, s))
self.assertEqual(len(stack.transactions), 0)
self.assertEqual(len(rcvErrors), 0)
self.assertEqual(received, expected)
def masterPeer(self, name, port, minionCount, action):
'''
Master peer main function. It creates a RoadStack, sends/receives a bunch of test messages to remote peers and
checks the result
:param name: the name of this peer
:param port: port to bind to
:param minionCount: count of remote peers
:param action: action function. Could be one of self.send, self.receive or self.bidirectional
:return: if done without errors the process will end with 0 exit code
'''
self.engine = False
# create stack
self.stack = self.createStack(name, port)
console.terse("\nCreated {0} at {1} *********\n".format(name, self.stack.ha))
console.terse("\n{0} Waiting For Minions *********\n".format(name))
# wait minions to join and allow
def isBootstrapDone():
if not self.stack.remotes:
return False
if len(self.stack.remotes) != minionCount:
return False
for rmt in self.stack.remotes.values():
if not rmt.allowed:
return False
return True
serviceCode = self.serviceOne(self.stack, timeout=100.0, exitCase=isBootstrapDone)
console.terse("\n{0} bootstrap done with code {1}".format(name, serviceCode))
# verify all minions connected
self.assertEqual(len(self.stack.transactions), 0)
self.assertEqual(len(self.stack.remotes), minionCount)
for remote in self.stack.remotes.values():
self.assertTrue(remote.joined)
self.assertTrue(remote.allowed)
console.terse("\n{0} Bootstrap Done ({1}) *********\n".format(name, serviceCode))
# do the job
action(self.stack, duration=self.duration)
def minionPeer(self, name, port, remoteAddresses, action):
'''
Slave peer main function. It creates a RoadStack, sends/receives a bunch of test messages to remote peers and
checks the result
:param name: the name of this peer
:param port: port to bind to
:param minionCount: count of remote peers
:param action: action function. Could be one of self.send, self.receive or self.bidirectional
:return: if done without errors the process will end with 0 exit code
'''
self.engine = False
# Create stack
self.stack = self.createStack(name, port)
console.terse("\nCreated {0} at {1} *********\n".format(name, self.stack.ha))
console.terse("\n{0} Joining Remotes *********\n".format(name))
# join with all masters
self.joinAll(self.stack, remoteAddresses, retryCount=10)
self.assertEqual(len(self.stack.transactions), 0)
self.assertEqual(len(self.stack.remotes), len(remoteAddresses))
for remote in self.stack.remotes.values():
self.assertTrue(remote.joined, "{0}: remote '{1}' at {2} is not joined".format(
name, remote.name, remote.ha))
console.terse("\n{0} Allowing Remotes *********\n".format(name))
# allow all masters
self.allowAll(self.stack, retryCount=10)
self.assertEqual(len(self.stack.transactions), 0)
self.assertEqual(len(self.stack.remotes), len(remoteAddresses))
for remote in self.stack.remotes.values():
self.assertTrue(remote.allowed, "{0}: remote '{1}' at {2} is not allowed".format(
name, remote.name, remote.ha))
console.terse("\n{0} Bootstrap Done *********\n".format(name))
# do the job
action(self.stack, duration=self.duration)
def messagingMultiPeers(self,
masterCount,
minionCount,
msgSize,
msgCount,
duration,
direction):
'''
Perform the test. This function creates a number of processes which are master and slave network peers. Each
process sends a bunch of messages to each process from the opposite group (for bidirectional case)
:param masterCount: count of master peers
:param minionCount: count of slave peers
:param msgSize: size of the message stuff, actually the message would be bigger
:param msgCount: count of messages to be sent from peer to peer
:param duration: test duration limit, will be used in service calls so actual allowed duration will be greater
:param direction: sending duration, could be one of DIR_TO_MASTER, DIR_FROM_MASTER or DIR_BIDIRECTIONAL
:return: throw PyUnit assertion if fail
'''
self.masterCount = masterCount
self.minionCount = minionCount
self.msgSize = msgSize
self.msgCount = msgCount
self.duration = duration
masterDir = {DIR_BIDIRECTIONAL: self.bidirectional,
DIR_TO_MASTER: self.receive,
DIR_FROM_MASTER: self.send}
minionDir = {DIR_BIDIRECTIONAL: self.bidirectional,
DIR_TO_MASTER: self.send,
DIR_FROM_MASTER: self.receive}
# create masters
port = raeting.RAET_PORT
masterHostAddresses = []
masterProcs = []
self.doneCounter = mp_helper.Counter()
self.stopFlag = mp_helper.Counter()
for i in xrange(masterCount):
masterHostAddresses.append(('', port))
name = 'master{0}'.format(i)
masterProc = multiprocessing.Process(target=self.masterPeer,
name=name,
args=(name,
port,
minionCount,
masterDir[direction]))
masterProcs.append(masterProc)
port += 1
# create minions
minionProcs = []
for i in xrange(minionCount):
name = 'minion{0}'.format(i)
minionProc = multiprocessing.Process(target=self.minionPeer,
name=name,
args=(name,
port,
masterHostAddresses,
minionDir[direction]))
minionProcs.append(minionProc)
port += 1
# start masters first
for proc in masterProcs:
proc.start()
time.sleep(1.0) # let masters start
# start minions
for proc in minionProcs:
proc.start()
# wait until all workers done their job
elapsed = 0.0 # be safe from worker unexpected stop
step = 1.0
while self.doneCounter.get() < masterCount + minionCount and elapsed < duration * 2:
time.sleep(step)
elapsed += step
# set stop flag
self.stopFlag.inc()
# wait all processes exited
for procs in (masterProcs, minionProcs):
for proc in procs:
proc.join()
# ensure all workers successfully done
for procs in (masterProcs, minionProcs):
for proc in procs:
self.assertEqual(proc.exitcode, 0, "'{0}' returned {1}".format(proc.name, proc.exitcode))
def setUp(self):
self.store = Store(stamp=0.0)
self.timer = StoreTimer(store=self.store, duration=1.0)
def __init__(
self,
store=None,
version=raeting.VERSION,
main=None,
puid=None,
local=None, #passed up from subclass
name='',
uid=None,
server=None,
ha=None,
bufcnt=2,
rxMsgs=None,
txMsgs=None,
rxes=None,
txes=None,
stats=None,
):
'''
Setup Stack instance
'''
self.store = store or Store(stamp=0.0)
self.version = version
self.main = main
if getattr(self, 'puid', None) is None:
self.puid = puid if puid is not None else self.Uid
self.local = local or lotting.Lot(
stack=self,
name=name,
uid=uid,
ha=ha,
)
self.local.stack = self
self.remotes = self.uidRemotes = odict() # remotes indexed by uid
self.nameRemotes = odict() # remotes indexed by name
self.bufcnt = bufcnt
if not server:
server = self.serverFromLocal()
self.server = server
if self.server:
if not self.server.reopen(): # open socket
raise raeting.StackError(
"Stack '{0}': Failed opening server at"
" '{1}'\n".format(self.name, self.server.ha))
self.ha = self.server.ha # update local host address after open
console.verbose("Stack '{0}': Opened server at '{1}'\n".format(
self.name, self.ha))
self.rxMsgs = rxMsgs if rxMsgs is not None else deque(
) # messages received
self.txMsgs = txMsgs if txMsgs is not None else deque(
) # messages to transmit
self.rxes = rxes if rxes is not None else deque(
) # udp packets received
self.txes = txes if txes is not None else deque(
) # udp packet to transmit
self.stats = stats if stats is not None else odict() # udp statistics
self.statTimer = StoreTimer(self.store)
def addDoneTransaction(self, index):
self.doneTransactions[index] = StoreTimer(
self.stack.store, duration=self.stack.MsgStaleTimeout)
class RemoteEstate(Estate):
'''
RAET protocol endpoint remote estate object ie Remote Road Lot
Maintains verifier for verifying signatures and publican for encrypt/decrypt
.alived attribute is the dead or alive status of the remote
.alived = True, alive, recently have received valid signed packets from remote
.alive = False, dead, recently have not received valid signed packets from remote
.fuid is the far uid of the remote as owned by the farside stack
'''
def __init__(self,
stack,
uid=None,
fuid=0,
main=False,
kind=0,
verkey=None,
pubkey=None,
acceptance=None,
joined=None,
rsid=0,
**kwa):
'''
Setup instance
stack is required parameter
verkey is either nacl VerifyKey or raw or hex encoded key
pubkey is either nacl PublicKey or raw or hex encoded key
acceptance is accepted state of remote on Road
rsid is last received session id used by remotely initiated transaction
'''
if uid is None:
uid = stack.nextUid()
while uid in stack.remotes or uid == stack.local.uid:
uid = stack.nextUid()
if 'ha' not in kwa:
kwa['ha'] = ('127.0.0.1', raeting.RAET_TEST_PORT)
super(RemoteEstate, self).__init__(stack, uid=uid, **kwa)
self.fuid = fuid
self.main = main
self.kind = kind
self.joined = joined
self.allowed = None
self.alived = None
self.reaped = None
self.acceptance = acceptance
self.privee = nacling.Privateer() # short term key manager
self.publee = nacling.Publican() # correspondent short term key manager
self.verfer = nacling.Verifier(verkey) # correspondent verify key manager
self.pubber = nacling.Publican(pubkey) # correspondent long term key manager
self.rsid = rsid # last sid received from remote when RmtFlag is True
# persistence keep alive heartbeat timer. Initial duration has offset so
# not synced with other side persistence heatbeet
# by default do not use offset on main
if self.stack.main:
duration = self.stack.period
else:
duration = self.stack.period + self.stack.offset
self.timer = StoreTimer(store=self.stack.store,
duration=duration)
self.reapTimer = StoreTimer(self.stack.store,
duration=self.stack.interim)
self.messages = deque() # deque of saved stale message body data to remote.uid
@property
def nuid(self):
'''
property that returns nuid, near uid, of remote as owned by nearside stack
alias for uid
'''
return self.uid
@nuid.setter
def nuid(self, value):
'''
setter for nuid, near uid, property
'''
self.uid = value
@property
def juid(self):
'''
property that returns juid, join uid, duple of (nuid, fuid)
nuid is near uid
fuid is far uid as owned by farside stack
'''
return (self.nuid, self.fuid)
@juid.setter
def juid(self, value):
'''
setter for juid, join uid, property, value is duple of (nuid, fuid)
'''
self.nuid, self.fuid = value
def rekey(self):
'''
Regenerate short term keys
'''
self.allowed = None
self.privee = nacling.Privateer() # short term key
self.publee = nacling.Publican() # correspondent short term key manager
def validRsid(self, rsid):
'''
Compare new rsid to old .rsid and return True
If old is zero Then new is always valid
If new is >= old modulo N where N is 2^32 = 0x100000000
And >= means the difference is less than N//2 = 0x80000000
(((new - old) % 0x100000000) < (0x100000000 // 2))
'''
return self.validateSid(new=rsid, old=self.rsid)
def refresh(self, alived=True):
'''
Restart presence heartbeat timer and conditionally reapTimer
If alived is None then do not change .alived but update timer
If alived is True then set .alived to True and handle implications
If alived is False the set .alived to False and handle implications
'''
self.timer.restart(duration=self.stack.period)
if alived is None:
return
if self.alived or alived: # alive before or after
self.reapTimer.restart()
if self.reaped:
self.unreap()
#otherwise let timer run both before and after are still dead
self.alived = alived
def manage(self, cascade=False, immediate=False):
'''
Perform time based processing of keep alive heatbeat
'''
if not self.reaped: # only manage alives if not already reaped
if immediate or self.timer.expired:
# alive transaction restarts self.timer
self.stack.alive(uid=self.uid, cascade=cascade)
if self.stack.interim > 0.0 and self.reapTimer.expired:
self.reap()
def reap(self):
'''
Remote is dead, reap it if main estate.
'''
if self.stack.main: # only main can reap
console.concise("Stack {0}: Reaping dead remote {1} at {2}\n".format(
self.stack.name, self.name, self.stack.store.stamp))
self.stack.incStat("remote_reap")
self.reaped = True
def unreap(self):
'''
Remote packet received from remote so not dead anymore.
'''
if self.stack.main: # only only main can reap or unreap
console.concise("Stack {0}: Unreaping dead remote {1} at {2}\n".format(
self.stack.name, self.name, self.stack.store.stamp))
self.stack.incStat("remote_unreap")
self.reaped = False
def removeStaleCorrespondents(self):
'''
Remove local stale correspondent transactions associated with remote
Local correspondent is indicated by rf ==True
Stale means the sid in the transaction is older than the current .rsid
assuming neither is zero, that is sid in index is older than remote.rsid
old rsid == 0 means new always valid
When sid in index is older than remote.rsid
Where index is tuple: (rf, le, re, si, ti, bf,)
rf = Remotely Initiated Flag, RmtFlag
le = leid, Local estate ID, LEID
re = reid, Remote estate ID, REID
si = sid, Session ID, SID
ti = tid, Transaction ID, TID
bf = Broadcast Flag, BcstFlag
'''
for index, transaction in self.transactions.items():
sid = index[3]
rf = index[0] # correspondent
if rf and not self.validRsid(sid):
transaction.nack()
self.removeTransaction(index)
emsg = ("Stack {0}: Stale correspondent {1} from remote {1} at {2}"
"\n".format(self.stack.name,
index,
self.name,
self.stack.store.stamp))
console.terse(emsg)
self.stack.incStat('stale_correspondent')
self.doneTransactions.clear()
def replaceStaleInitiators(self):
'''
Save and remove any messages from messenger transactions initiated locally
with remote
Remove non message stale local initiator transactions associated with remote
Also save and requeue any stale locally initiated message transactions.
Local inititors have rf flag == False
Stale means the sid in the transaction is older than the current .sid
assuming neither is zero, that is sid in index is older than remote.sid
old sid == 0 means new always valid
Where index is tuple: (rf, le, re, si, ti, bf,)
rf = Remotely Initiated Flag, RmtFlag
le = leid, Local estate ID, LEID
re = reid, Remote estate ID, REID
si = sid, Session ID, SID
ti = tid, Transaction ID, TID
bf = Broadcast Flag, BcstFlag
'''
for index, transaction in self.transactions.items():
rf = index[0]
sid = index[3]
if not rf and not self.validSid(sid): # transaction sid newer or equal
if transaction.kind in [TrnsKind.message]:
self.saveMessage(transaction)
transaction.nack()
self.removeTransaction(index)
emsg = ("Stack {0}: Stale initiator {1} to remote {2} at {3}"
"\n".format(self.stack.name,
index,
self.name,
self.stack.store.stamp))
console.terse(emsg)
self.stack.incStat('stale_initiator')
def saveMessage(self, messenger):
'''
Save copy of body data from stale initiated messenger onto .messages deque
for retransmitting later after new session is established
messenger is instance of Messenger compatible transaction
'''
self.messages.append(odict(messenger.tray.body))
emsg = ("Stack {0}: Saved stale message with remote {1}"
"\n".format(self.stack.name,
self.name))
console.concise(emsg)
def sendSavedMessages(self):
'''
Message is Messenger compatible transaction
Save stale initiated message for retransmitting later after new session is established
'''
while self.messages:
body = self.messages.popleft()
self.stack.message(body, uid=self.uid)
emsg = ("Stack {0}: Resent saved message with remote {1}"
"\n".format(self.stack.name, self.name))
console.concise(emsg)
def allowInProcess(self):
'''
Returns list of transactions for all allow transactions with this remote
that are already in process
'''
return ([t for t in self.transactions.values()
if t.kind == TrnsKind.allow])
def joinInProcess(self):
'''
Returns list of transactions for all join transaction with this remote
that are already in process
'''
return ([t for t in self.transactions.values()
if t.kind == TrnsKind.join])
def __init__(self,
stack,
uid=None,
fuid=0,
main=False,
kind=0,
verkey=None,
pubkey=None,
acceptance=None,
joined=None,
rsid=0,
**kwa):
'''
Setup instance
stack is required parameter
verkey is either nacl VerifyKey or raw or hex encoded key
pubkey is either nacl PublicKey or raw or hex encoded key
acceptance is accepted state of remote on Road
rsid is last received session id used by remotely initiated transaction
'''
if uid is None:
uid = stack.nextUid()
while uid in stack.remotes or uid == stack.local.uid:
uid = stack.nextUid()
if 'ha' not in kwa:
kwa['ha'] = ('127.0.0.1', raeting.RAET_TEST_PORT)
super(RemoteEstate, self).__init__(stack, uid=uid, **kwa)
self.fuid = fuid
self.main = main
self.kind = kind
self.joined = joined
self.allowed = None
self.alived = None
self.reaped = None
self.acceptance = acceptance
self.privee = nacling.Privateer() # short term key manager
self.publee = nacling.Publican(
) # correspondent short term key manager
self.verfer = nacling.Verifier(
verkey) # correspondent verify key manager
self.pubber = nacling.Publican(
pubkey) # correspondent long term key manager
self.rsid = rsid # last sid received from remote when RmtFlag is True
# persistence keep alive heartbeat timer. Initial duration has offset so
# not synced with other side persistence heatbeet
# by default do not use offset on main
if self.stack.main:
duration = self.stack.period
else:
duration = self.stack.period + self.stack.offset
self.timer = StoreTimer(store=self.stack.store, duration=duration)
self.reapTimer = StoreTimer(self.stack.store,
duration=self.stack.interim)
self.messages = deque(
) # deque of saved stale message body data to remote.uid
def __init__(self,
stack,
uid=None,
fuid=0,
main=False,
kind=0,
verkey=None,
pubkey=None,
acceptance=None,
joined=None,
rsid=0,
**kwa):
'''
Setup instance
stack is required parameter
verkey is either nacl VerifyKey or raw or hex encoded key
pubkey is either nacl PublicKey or raw or hex encoded key
acceptance is accepted state of remote on Road
rsid is last received session id used by remotely initiated transaction
'''
if uid is None:
uid = stack.nextUid()
while uid in stack.remotes or uid == stack.local.uid:
uid = stack.nextUid()
if 'ha' not in kwa:
kwa['ha'] = ('127.0.0.1', raeting.RAET_TEST_PORT)
super(RemoteEstate, self).__init__(stack, uid=uid, **kwa)
self.fuid = fuid
self.main = main
self.kind = kind
self.joined = joined
self.allowed = None
self.alived = None
self.reaped = None
self.acceptance = acceptance
self.privee = nacling.Privateer() # short term key manager
self.publee = nacling.Publican() # correspondent short term key manager
self.verfer = nacling.Verifier(verkey) # correspondent verify key manager
self.pubber = nacling.Publican(pubkey) # correspondent long term key manager
self.rsid = rsid # last sid received from remote when RmtFlag is True
# persistence keep alive heartbeat timer. Initial duration has offset so
# not synced with other side persistence heatbeet
# by default do not use offset on main
if self.stack.main:
duration = self.stack.period
else:
duration = self.stack.period + self.stack.offset
self.timer = StoreTimer(store=self.stack.store,
duration=duration)
self.reapTimer = StoreTimer(self.stack.store,
duration=self.stack.interim)
self.messages = deque() # deque of saved stale message body data to remote.uid
def testValetServiceBottleStream(self):
"""
Test Valet WSGI service request response stream sse
"""
console.terse("{0}\n".format(
self.testValetServiceBottleStream.__doc__))
try:
import bottle
except ImportError as ex:
console.terse("Bottle not available.\n")
return
store = storing.Store(stamp=0.0)
app = bottle.default_app() # create bottle app
@app.get('/stream')
def streamGet():
"""
Create test server sent event stream that sends count events
"""
timer = StoreTimer(store, duration=2.0)
bottle.response.set_header('Content-Type',
'text/event-stream') #text
bottle.response.set_header('Cache-Control', 'no-cache')
# HTTP 1.1 servers detect text/event-stream and use Transfer-Encoding: chunked
# Set client-side auto-reconnect timeout, ms.
yield 'retry: 1000\n\n'
i = 0
yield 'id: {0}\n'.format(i)
i += 1
yield 'data: START\n\n'
n = 1
while not timer.expired:
yield 'id: {0}\n'.format(i)
i += 1
yield 'data: {0}\n\n'.format(n)
n += 1
yield "data: END\n\n"
console.terse("{0}\n".format("Building Valet ...\n"))
wireLogAlpha = wiring.WireLog(buffify=True, same=True)
result = wireLogAlpha.reopen()
alpha = serving.Valet(port=6101,
bufsize=131072,
wlog=wireLogAlpha,
store=store,
app=app)
self.assertIs(alpha.servant.reopen(), True)
self.assertEqual(alpha.servant.ha, ('0.0.0.0', 6101))
self.assertEqual(alpha.servant.eha, ('127.0.0.1', 6101))
console.terse("{0}\n".format("Building Patron ...\n"))
wireLogBeta = wiring.WireLog(buffify=True, same=True)
result = wireLogBeta.reopen()
path = "http://{0}:{1}/".format('localhost', alpha.servant.eha[1])
beta = clienting.Patron(
bufsize=131072,
wlog=wireLogBeta,
store=store,
path=path,
reconnectable=True,
)
self.assertIs(beta.connector.reopen(), True)
self.assertIs(beta.connector.accepted, False)
self.assertIs(beta.connector.connected, False)
self.assertIs(beta.connector.cutoff, False)
request = odict([
('method', u'GET'),
('path', u'/stream'),
('qargs', odict()),
('fragment', u''),
('headers',
odict([('Accept', 'application/json'), ('Content-Length', 0)])),
('body', None),
])
beta.requests.append(request)
timer = StoreTimer(store, duration=1.0)
while (not timer.expired):
alpha.serviceAll()
time.sleep(0.05)
beta.serviceAll()
time.sleep(0.05)
store.advanceStamp(0.1)
self.assertIs(beta.connector.accepted, True)
self.assertIs(beta.connector.connected, True)
self.assertIs(beta.connector.cutoff, False)
self.assertEqual(len(alpha.servant.ixes), 1)
self.assertEqual(len(alpha.reqs), 1)
self.assertEqual(len(alpha.reps), 1)
requestant = alpha.reqs.values()[0]
self.assertEqual(requestant.method, request['method'])
self.assertEqual(requestant.url, request['path'])
self.assertEqual(
requestant.headers, {
'accept': 'application/json',
'accept-encoding': 'identity',
'content-length': '0',
'host': 'localhost:6101'
})
#timed out while stream still open so no responses in .responses
self.assertIs(beta.waited, True)
self.assertIs(beta.respondent.ended, False)
self.assertEqual(len(beta.responses), 0)
self.assertIn('content-type', beta.respondent.headers)
self.assertEqual(beta.respondent.headers['content-type'],
'text/event-stream')
self.assertIn('transfer-encoding', beta.respondent.headers)
self.assertEqual(beta.respondent.headers['transfer-encoding'],
'chunked')
self.assertTrue(len(beta.events) >= 3)
self.assertEqual(beta.respondent.retry, 1000)
self.assertTrue(int(beta.respondent.leid) >= 2)
event = beta.events.popleft()
self.assertEqual(event, {'id': '0', 'name': '', 'data': 'START'})
event = beta.events.popleft()
self.assertEqual(event, {'id': '1', 'name': '', 'data': '1'})
event = beta.events.popleft()
self.assertEqual(event, {'id': '2', 'name': '', 'data': '2'})
beta.events.clear()
#keep going until ended
timer.restart(duration=1.5)
while (not timer.expired):
alpha.serviceAll()
time.sleep(0.05)
beta.serviceAll()
time.sleep(0.05)
store.advanceStamp(0.1)
self.assertTrue(len(beta.events) >= 3)
self.assertEqual(beta.respondent.leid, '9')
self.assertEqual(beta.events[-2], {'id': '9', 'name': '', 'data': '9'})
self.assertEqual(beta.events[-1], {
'id': '9',
'name': '',
'data': 'END'
})
beta.events.clear()
alpha.servant.closeAll()
beta.connector.close()
wireLogAlpha.close()
wireLogBeta.close()
def testTestStream(self):
"""
Test Valet WSGI service request response stream sse
"""
console.terse("{0}\n".format(self.testTestStream.__doc__))
store = Store(stamp=0.0)
app = bottle.default_app() # create bottle app
ending.loadTest(app, store)
console.terse("{0}\n".format("Building Valet ...\n"))
wireLogAlpha = WireLog(buffify=True, same=True)
result = wireLogAlpha.reopen()
alpha = Valet(port = 6101,
bufsize=131072,
wlog=wireLogAlpha,
store=store,
app=app)
self.assertIs(alpha.servant.reopen(), True)
self.assertEqual(alpha.servant.ha, ('0.0.0.0', 6101))
self.assertEqual(alpha.servant.eha, ('127.0.0.1', 6101))
console.terse("{0}\n".format("Building Patron ...\n"))
wireLogBeta = WireLog(buffify=True, same=True)
result = wireLogBeta.reopen()
path = "http://{0}:{1}/".format('localhost', alpha.servant.eha[1])
beta = Patron(bufsize=131072,
wlog=wireLogBeta,
store=store,
path=path,
reconnectable=True)
self.assertIs(beta.connector.reopen(), True)
self.assertIs(beta.connector.accepted, False)
self.assertIs(beta.connector.connected, False)
self.assertIs(beta.connector.cutoff, False)
request = odict([('method', u'GET'),
('path', u'/test/stream'),
('qargs', odict()),
('fragment', u''),
('headers', odict([('Accept', 'application/json'),
('Content-Length', 0)])),
('body', None),
])
beta.requests.append(request)
timer = StoreTimer(store, duration=1.0)
while (not timer.expired):
alpha.serviceAll()
time.sleep(0.05)
beta.serviceAll()
time.sleep(0.05)
store.advanceStamp(0.1)
self.assertIs(beta.connector.accepted, True)
self.assertIs(beta.connector.connected, True)
self.assertIs(beta.connector.cutoff, False)
self.assertEqual(len(alpha.servant.ixes), 1)
self.assertEqual(len(alpha.reqs), 1)
self.assertEqual(len(alpha.reps), 1)
requestant = alpha.reqs.values()[0]
self.assertEqual(requestant.method, request['method'])
self.assertEqual(requestant.url, request['path'])
self.assertEqual(requestant.headers, {'accept': 'application/json',
'accept-encoding': 'identity',
'content-length': '0',
'host': 'localhost:6101'})
#timed out while stream still open so no responses in .responses
self.assertIs(beta.waited, True)
self.assertIs(beta.respondent.ended, False)
self.assertEqual(len(beta.responses), 0)
self.assertIn('content-type', beta.respondent.headers)
self.assertEqual(beta.respondent.headers['content-type'], 'text/event-stream')
self.assertIn('transfer-encoding', beta.respondent.headers)
self.assertEqual(beta.respondent.headers['transfer-encoding'], 'chunked')
self.assertTrue(len(beta.events) >= 3)
self.assertEqual(beta.respondent.retry, 1000)
self.assertTrue(int(beta.respondent.leid) >= 2)
event = beta.events.popleft()
self.assertEqual(event, {'id': '0', 'name': '', 'data': 'START'})
event = beta.events.popleft()
self.assertEqual(event, {'id': '1', 'name': '', 'data': '1'})
event = beta.events.popleft()
self.assertEqual(event, {'id': '2', 'name': '', 'data': '2'})
beta.events.clear()
#keep going until ended
timer.restart(duration=1.5)
while (not timer.expired):
alpha.serviceAll()
time.sleep(0.05)
beta.serviceAll()
time.sleep(0.05)
store.advanceStamp(0.1)
self.assertTrue(len(beta.events) >= 3)
self.assertEqual(beta.respondent.leid, '9')
self.assertEqual(beta.events[-2], {'id': '9', 'name': '', 'data': '9'})
self.assertEqual(beta.events[-1], {'id': '9', 'name': '', 'data': 'END'})
beta.events.clear()
alpha.servant.closeAll()
beta.connector.close()
wireLogAlpha.close()
wireLogBeta.close()
