def test_socket_listener(self):
'''
A simple test to check that SocketListenerApps are indeed working as
expected; that is, they write the data they receive into their output,
and finish when the connection is closed from the client side
The data flow diagram looks like this:
A --> B --> C --> D
'''
host = 'localhost'
port = 9933
data = 'shine on you crazy diamond'
a = SocketListenerApp('oid:A', 'uid:A', host=host, port=port)
b = InMemoryDROP('oid:B', 'uid:B')
c = SumupContainerChecksum('oid:C', 'uid:C')
d = InMemoryDROP('oid:D', 'uid:D')
a.addOutput(b)
b.addConsumer(c)
c.addOutput(d)
# Create the socket, write, and close the connection, allowing
# A to move to COMPLETED
with DROPWaiterCtx(self, d, 3): # That's plenty of time
threading.Thread(target=lambda a: a.execute(), args=(a,)).start()
utils.writeToRemotePort(host, port, data, 1)
for drop in [a,b,c,d]:
self.assertEquals(DROPStates.COMPLETED, drop.status)
# Our expectations are fulfilled!
bContents = droputils.allDropContents(b)
dContents = int(droputils.allDropContents(d))
self.assertEquals(data, bContents)
self.assertEquals(crc32(data, 0), dContents)
def test_fullRound(self):
"""
A test that exercises most of the REST interface exposed on top of the
NodeManager
"""
sessionId = 'lala'
restPort = 8888
args = [sys.executable, '-m', 'dfms.manager.cmdline', 'dfmsNM', \
'--port', str(restPort), '-qqq']
dmProcess = subprocess.Popen(args)
with testutils.terminating(dmProcess, 10):
# Wait until the REST server becomes alive
self.assertTrue(utils.portIsOpen('localhost', restPort, 10), "REST server didn't come up in time")
# The DM is still empty
dmInfo = testutils.get(self, '', restPort)
self.assertEquals(0, len(dmInfo['sessions']))
# Create a session and check it exists
testutils.post(self, '/sessions', restPort, '{"sessionId":"%s"}' % (sessionId))
dmInfo = testutils.get(self, '', restPort)
self.assertEquals(1, len(dmInfo['sessions']))
self.assertEquals(sessionId, dmInfo['sessions'][0]['sessionId'])
self.assertEquals(SessionStates.PRISTINE, dmInfo['sessions'][0]['status'])
# Add this complex graph spec to the session
# The UID of the two leaf nodes of this complex.js graph are T and S
# PRO-242: use timestamps for final DROPs that get archived into the public NGAS
graph = json.loads(pkg_resources.resource_string('test', 'graphs/complex.js')) # @UndefinedVariable
suffix = '_' + str(int(time.time()))
oidsToReplace = ('S','T')
for dropSpec in graph:
if dropSpec['oid'] in oidsToReplace:
dropSpec['oid'] += suffix
for rel in ('inputs','outputs'):
if rel in dropSpec:
for oid in dropSpec[rel][:]:
if oid in oidsToReplace:
dropSpec[rel].remove(oid)
dropSpec[rel].append(oid + suffix)
testutils.post(self, '/sessions/%s/graph/append' % (sessionId), restPort, json.dumps(graph))
# We create two final archiving nodes, but this time from a template
# available on the server-side
timeout = 10
testutils.post(self, '/templates/dfms.manager.repository.archiving_app/materialize?uid=archiving1&host=ngas.ddns.net&port=7777&sessionId=%s&connect_timeout=%f&timeout=%f' % (sessionId, timeout, timeout), restPort)
testutils.post(self, '/templates/dfms.manager.repository.archiving_app/materialize?uid=archiving2&host=ngas.ddns.net&port=7777&sessionId=%s&connect_timeout=%f&timeout=%f' % (sessionId, timeout, timeout), restPort)
# And link them to the leaf nodes of the complex graph
testutils.post(self, '/sessions/%s/graph/link?rhOID=archiving1&lhOID=S%s&linkType=0' % (sessionId, suffix), restPort)
testutils.post(self, '/sessions/%s/graph/link?rhOID=archiving2&lhOID=T%s&linkType=0' % (sessionId, suffix), restPort)
# Now we deploy the graph...
testutils.post(self, '/sessions/%s/deploy' % (sessionId), restPort, 'completed=SL_A,SL_B,SL_C,SL_D,SL_K', mimeType='application/x-www-form-urlencoded')
# ...and write to all 5 root nodes that are listening in ports
# starting at 1111
msg = ''.join([random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in xrange(10)])
for i in xrange(5):
self.assertTrue(utils.writeToRemotePort('localhost', 1111+i, msg, 2), "Couldn't write data to localhost:%d" % (1111+i))
# Wait until the graph has finished its execution. We'll know
# it finished by polling the status of the session
while testutils.get(self, '/sessions/%s/status' % (sessionId), restPort) == SessionStates.RUNNING:
time.sleep(0.2)
self.assertEquals(SessionStates.FINISHED, testutils.get(self, '/sessions/%s/status' % (sessionId), restPort))
testutils.delete(self, '/sessions/%s' % (sessionId), restPort)
# We put an NGAS archiving at the end of the chain, let's check that the DROPs were copied over there
# Since the graph consists on several SleepAndCopy apps, T should contain the message repeated
# 9 times, and S should have it 4 times
def checkReplica(dropId, copies):
response = ngaslite.retrieve('ngas.ddns.net', dropId)
buff = response.read()
self.assertEquals(msg*copies, buff, "%s's replica doesn't look correct" % (dropId))
checkReplica('T%s' % (suffix), 9)
checkReplica('S%s' % (suffix), 4)
