def test_socket_listener(self): ''' A simple test to check that SocketListenerApps are indeed working as expected; that is, they write the data they receive into their output, and finish when the connection is closed from the client side The data flow diagram looks like this: A --> B --> C --> D ''' host = 'localhost' port = 9933 data = 'shine on you crazy diamond' a = SocketListenerApp('oid:A', 'uid:A', host=host, port=port) b = InMemoryDROP('oid:B', 'uid:B') c = SumupContainerChecksum('oid:C', 'uid:C') d = InMemoryDROP('oid:D', 'uid:D') a.addOutput(b) b.addConsumer(c) c.addOutput(d) # Create the socket, write, and close the connection, allowing # A to move to COMPLETED with DROPWaiterCtx(self, d, 3): # That's plenty of time threading.Thread(target=lambda a: a.execute(), args=(a,)).start() utils.writeToRemotePort(host, port, data, 1) for drop in [a,b,c,d]: self.assertEquals(DROPStates.COMPLETED, drop.status) # Our expectations are fulfilled! bContents = droputils.allDropContents(b) dContents = int(droputils.allDropContents(d)) self.assertEquals(data, bContents) self.assertEquals(crc32(data, 0), dContents)
def test_fullRound(self): """ A test that exercises most of the REST interface exposed on top of the NodeManager """ sessionId = 'lala' restPort = 8888 args = [sys.executable, '-m', 'dfms.manager.cmdline', 'dfmsNM', \ '--port', str(restPort), '-qqq'] dmProcess = subprocess.Popen(args) with testutils.terminating(dmProcess, 10): # Wait until the REST server becomes alive self.assertTrue(utils.portIsOpen('localhost', restPort, 10), "REST server didn't come up in time") # The DM is still empty dmInfo = testutils.get(self, '', restPort) self.assertEquals(0, len(dmInfo['sessions'])) # Create a session and check it exists testutils.post(self, '/sessions', restPort, '{"sessionId":"%s"}' % (sessionId)) dmInfo = testutils.get(self, '', restPort) self.assertEquals(1, len(dmInfo['sessions'])) self.assertEquals(sessionId, dmInfo['sessions'][0]['sessionId']) self.assertEquals(SessionStates.PRISTINE, dmInfo['sessions'][0]['status']) # Add this complex graph spec to the session # The UID of the two leaf nodes of this complex.js graph are T and S # PRO-242: use timestamps for final DROPs that get archived into the public NGAS graph = json.loads(pkg_resources.resource_string('test', 'graphs/complex.js')) # @UndefinedVariable suffix = '_' + str(int(time.time())) oidsToReplace = ('S','T') for dropSpec in graph: if dropSpec['oid'] in oidsToReplace: dropSpec['oid'] += suffix for rel in ('inputs','outputs'): if rel in dropSpec: for oid in dropSpec[rel][:]: if oid in oidsToReplace: dropSpec[rel].remove(oid) dropSpec[rel].append(oid + suffix) testutils.post(self, '/sessions/%s/graph/append' % (sessionId), restPort, json.dumps(graph)) # We create two final archiving nodes, but this time from a template # available on the server-side timeout = 10 testutils.post(self, '/templates/dfms.manager.repository.archiving_app/materialize?uid=archiving1&host=ngas.ddns.net&port=7777&sessionId=%s&connect_timeout=%f&timeout=%f' % (sessionId, timeout, timeout), restPort) testutils.post(self, '/templates/dfms.manager.repository.archiving_app/materialize?uid=archiving2&host=ngas.ddns.net&port=7777&sessionId=%s&connect_timeout=%f&timeout=%f' % (sessionId, timeout, timeout), restPort) # And link them to the leaf nodes of the complex graph testutils.post(self, '/sessions/%s/graph/link?rhOID=archiving1&lhOID=S%s&linkType=0' % (sessionId, suffix), restPort) testutils.post(self, '/sessions/%s/graph/link?rhOID=archiving2&lhOID=T%s&linkType=0' % (sessionId, suffix), restPort) # Now we deploy the graph... testutils.post(self, '/sessions/%s/deploy' % (sessionId), restPort, 'completed=SL_A,SL_B,SL_C,SL_D,SL_K', mimeType='application/x-www-form-urlencoded') # ...and write to all 5 root nodes that are listening in ports # starting at 1111 msg = ''.join([random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in xrange(10)]) for i in xrange(5): self.assertTrue(utils.writeToRemotePort('localhost', 1111+i, msg, 2), "Couldn't write data to localhost:%d" % (1111+i)) # Wait until the graph has finished its execution. We'll know # it finished by polling the status of the session while testutils.get(self, '/sessions/%s/status' % (sessionId), restPort) == SessionStates.RUNNING: time.sleep(0.2) self.assertEquals(SessionStates.FINISHED, testutils.get(self, '/sessions/%s/status' % (sessionId), restPort)) testutils.delete(self, '/sessions/%s' % (sessionId), restPort) # We put an NGAS archiving at the end of the chain, let's check that the DROPs were copied over there # Since the graph consists on several SleepAndCopy apps, T should contain the message repeated # 9 times, and S should have it 4 times def checkReplica(dropId, copies): response = ngaslite.retrieve('ngas.ddns.net', dropId) buff = response.read() self.assertEquals(msg*copies, buff, "%s's replica doesn't look correct" % (dropId)) checkReplica('T%s' % (suffix), 9) checkReplica('S%s' % (suffix), 4)