Ejemplo n.º 1
0
    def test_fullRound(self):
        """
        A test that exercises most of the REST interface exposed on top of the
        DataIslandManager
        """

        sessionId = "lala"
        restPort = 8989  # don't interfere with EAGLE default port
        args = ["--port", str(restPort), "-N", hostname, "-qqq"]
        dimProcess = tool.start_process("dim", args)

        with testutils.terminating(dimProcess, timeout=10):

            # Wait until the REST server becomes alive
            self.assertTrue(
                utils.portIsOpen("localhost", restPort, timeout=10),
                "REST server didn't come up in time",
            )

            # The DIM is still empty
            sessions = testutils.get(self, "/sessions", restPort)
            self.assertEqual(0, len(sessions))
            dimStatus = testutils.get(self, "", restPort)
            self.assertEqual(1, len(dimStatus["hosts"]))
            self.assertEqual(hostname, dimStatus["hosts"][0])
            self.assertEqual(0, len(dimStatus["sessionIds"]))

            # Create a session and check it exists
            testutils.post(
                self, "/sessions", restPort, '{"sessionId":"%s"}' % (sessionId)
            )
            sessions = testutils.get(self, "/sessions", restPort)
            self.assertEqual(1, len(sessions))
            self.assertEqual(sessionId, sessions[0]["sessionId"])
            self.assertDictEqual(
                {hostname: SessionStates.PRISTINE}, sessions[0]["status"]
            )

            # Add this complex graph spec to the session
            # The UID of the two leaf nodes of this complex.js graph are T and S
            # Since the original complexGraph doesn't have node information
            # we need to add it manually before submitting -- otherwise it will
            # get rejected by the DIM.
            with pkg_resources.resource_stream(
                "test", "graphs/complex.js"
            ) as f:  # @UndefinedVariable
                complexGraphSpec = json.load(codecs.getreader("utf-8")(f))
                logger.debug(f"Loaded graph: {f}")
            for dropSpec in complexGraphSpec:
                dropSpec["node"] = hostname
            testutils.post(
                self,
                "/sessions/%s/graph/append" % (sessionId),
                restPort,
                json.dumps(complexGraphSpec),
            )
            self.assertEqual(
                {hostname: SessionStates.BUILDING},
                testutils.get(self, "/sessions/%s/status" % (sessionId), restPort),
            )

            # Now we deploy the graph...
            testutils.post(
                self,
                "/sessions/%s/deploy" % (sessionId),
                restPort,
                "completed=SL_A,SL_B,SL_C,SL_D,SL_K",
                mimeType="application/x-www-form-urlencoded",
            )
            self.assertEqual(
                {hostname: SessionStates.RUNNING},
                testutils.get(self, "/sessions/%s/status" % (sessionId), restPort),
            )

            # ...and write to all 5 root nodes that are listening in ports
            # starting at 1111
            msg = os.urandom(10)
            for i in range(5):
                utils.write_to(
                    "localhost", 1111 + i, msg, 2
                ), "Couldn't write data to localhost:%d" % (1111 + i)

            # Wait until the graph has finished its execution. We'll know
            # it finished by polling the status of the session
            while (
                SessionStates.RUNNING
                in testutils.get(
                    self, "/sessions/%s/status" % (sessionId), restPort
                ).values()
            ):
                time.sleep(0.2)

            self.assertEqual(
                {hostname: SessionStates.FINISHED},
                testutils.get(self, "/sessions/%s/status" % (sessionId), restPort),
            )
            testutils.delete(self, "/sessions/%s" % (sessionId), restPort)
            sessions = testutils.get(self, "/sessions", restPort)
            self.assertEqual(0, len(sessions))
Ejemplo n.º 2
0
    def test_fullRound(self):
        """
        A test that exercises most of the REST interface exposed on top of the
        DataIslandManager
        """

        sessionId = 'lala'
        restPort = 8888
        args = ['--port', str(restPort), '-N', hostname, '-qqq']
        dimProcess = tool.start_process('dim', args)

        with testutils.terminating(dimProcess, 10):

            # Wait until the REST server becomes alive
            self.assertTrue(utils.portIsOpen('localhost', restPort, 10),
                            "REST server didn't come up in time")

            # The DIM is still empty
            sessions = testutils.get(self, '/sessions', restPort)
            self.assertEqual(0, len(sessions))
            dimStatus = testutils.get(self, '', restPort)
            self.assertEqual(1, len(dimStatus['hosts']))
            self.assertEqual(hostname, dimStatus['hosts'][0])
            self.assertEqual(0, len(dimStatus['sessionIds']))

            # Create a session and check it exists
            testutils.post(self, '/sessions', restPort,
                           '{"sessionId":"%s"}' % (sessionId))
            sessions = testutils.get(self, '/sessions', restPort)
            self.assertEqual(1, len(sessions))
            self.assertEqual(sessionId, sessions[0]['sessionId'])
            self.assertDictEqual({hostname: SessionStates.PRISTINE},
                                 sessions[0]['status'])

            # Add this complex graph spec to the session
            # The UID of the two leaf nodes of this complex.js graph are T and S
            # Since the original complexGraph doesn't have node information
            # we need to add it manually before submitting -- otherwise it will
            # get rejected by the DIM.
            with pkg_resources.resource_stream(
                    'test', 'graphs/complex.js') as f:  # @UndefinedVariable
                complexGraphSpec = json.load(codecs.getreader('utf-8')(f))
            for dropSpec in complexGraphSpec:
                dropSpec['node'] = hostname
            testutils.post(self, '/sessions/%s/graph/append' % (sessionId),
                           restPort, json.dumps(complexGraphSpec))
            self.assertEqual({hostname: SessionStates.BUILDING},
                             testutils.get(self,
                                           '/sessions/%s/status' % (sessionId),
                                           restPort))

            # Now we deploy the graph...
            testutils.post(self,
                           '/sessions/%s/deploy' % (sessionId),
                           restPort,
                           "completed=SL_A,SL_B,SL_C,SL_D,SL_K",
                           mimeType='application/x-www-form-urlencoded')
            self.assertEqual({hostname: SessionStates.RUNNING},
                             testutils.get(self,
                                           '/sessions/%s/status' % (sessionId),
                                           restPort))

            # ...and write to all 5 root nodes that are listening in ports
            # starting at 1111
            msg = os.urandom(10)
            for i in range(5):
                utils.write_to(
                    'localhost', 1111 + i, msg,
                    2), "Couldn't write data to localhost:%d" % (1111 + i)

            # Wait until the graph has finished its execution. We'll know
            # it finished by polling the status of the session
            while SessionStates.RUNNING in testutils.get(
                    self, '/sessions/%s/status' % (sessionId),
                    restPort).values():
                time.sleep(0.2)

            self.assertEqual({hostname: SessionStates.FINISHED},
                             testutils.get(self,
                                           '/sessions/%s/status' % (sessionId),
                                           restPort))
            testutils.delete(self, '/sessions/%s' % (sessionId), restPort)
            sessions = testutils.get(self, '/sessions', restPort)
            self.assertEqual(0, len(sessions))
Ejemplo n.º 3
0
    def test_fullRound(self):
        """
        A test that exercises most of the REST interface exposed on top of the
        NodeManager
        """

        sessionId = 'lala'
        restPort  = 8888

        args = [sys.executable, '-m', 'dfms.manager.cmdline', 'dfmsNM', \
                '--port', str(restPort), '-qqq']
        dmProcess = subprocess.Popen(args)

        with testutils.terminating(dmProcess, 10):

            # Wait until the REST server becomes alive
            self.assertTrue(utils.portIsOpen('localhost', restPort, 10), "REST server didn't come up in time")

            # The DM is still empty
            dmInfo = testutils.get(self, '', restPort)
            self.assertEquals(0, len(dmInfo['sessions']))

            # Create a session and check it exists
            testutils.post(self, '/sessions', restPort, '{"sessionId":"%s"}' % (sessionId))
            dmInfo = testutils.get(self, '', restPort)
            self.assertEquals(1, len(dmInfo['sessions']))
            self.assertEquals(sessionId, dmInfo['sessions'][0]['sessionId'])
            self.assertEquals(SessionStates.PRISTINE, dmInfo['sessions'][0]['status'])

            # Add this complex graph spec to the session
            # The UID of the two leaf nodes of this complex.js graph are T and S
            # PRO-242: use timestamps for final DROPs that get archived into the public NGAS
            graph = json.loads(pkg_resources.resource_string('test', 'graphs/complex.js')) # @UndefinedVariable
            suffix = '_' + str(int(time.time()))
            oidsToReplace = ('S','T')
            for dropSpec in graph:
                if dropSpec['oid'] in oidsToReplace:
                    dropSpec['oid'] += suffix
                for rel in ('inputs','outputs'):
                    if rel in dropSpec:
                        for oid in dropSpec[rel][:]:
                            if oid in oidsToReplace:
                                dropSpec[rel].remove(oid)
                                dropSpec[rel].append(oid + suffix)

            testutils.post(self, '/sessions/%s/graph/append' % (sessionId), restPort, json.dumps(graph))

            # We create two final archiving nodes, but this time from a template
            # available on the server-side
            timeout = 10
            testutils.post(self, '/templates/dfms.manager.repository.archiving_app/materialize?uid=archiving1&host=ngas.ddns.net&port=7777&sessionId=%s&connect_timeout=%f&timeout=%f' % (sessionId, timeout, timeout), restPort)
            testutils.post(self, '/templates/dfms.manager.repository.archiving_app/materialize?uid=archiving2&host=ngas.ddns.net&port=7777&sessionId=%s&connect_timeout=%f&timeout=%f' % (sessionId, timeout, timeout), restPort)

            # And link them to the leaf nodes of the complex graph
            testutils.post(self, '/sessions/%s/graph/link?rhOID=archiving1&lhOID=S%s&linkType=0' % (sessionId, suffix), restPort)
            testutils.post(self, '/sessions/%s/graph/link?rhOID=archiving2&lhOID=T%s&linkType=0' % (sessionId, suffix), restPort)

            # Now we deploy the graph...
            testutils.post(self, '/sessions/%s/deploy' % (sessionId), restPort, 'completed=SL_A,SL_B,SL_C,SL_D,SL_K', mimeType='application/x-www-form-urlencoded')

            # ...and write to all 5 root nodes that are listening in ports
            # starting at 1111
            msg = ''.join([random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in xrange(10)])
            for i in xrange(5):
                self.assertTrue(utils.writeToRemotePort('localhost', 1111+i, msg, 2), "Couldn't write data to localhost:%d" % (1111+i))

            # Wait until the graph has finished its execution. We'll know
            # it finished by polling the status of the session
            while testutils.get(self, '/sessions/%s/status' % (sessionId), restPort) == SessionStates.RUNNING:
                time.sleep(0.2)

            self.assertEquals(SessionStates.FINISHED, testutils.get(self, '/sessions/%s/status' % (sessionId), restPort))
            testutils.delete(self, '/sessions/%s' % (sessionId), restPort)

            # We put an NGAS archiving at the end of the chain, let's check that the DROPs were copied over there
            # Since the graph consists on several SleepAndCopy apps, T should contain the message repeated
            # 9 times, and S should have it 4 times
            def checkReplica(dropId, copies):
                response = ngaslite.retrieve('ngas.ddns.net', dropId)
                buff = response.read()
                self.assertEquals(msg*copies, buff, "%s's replica doesn't look correct" % (dropId))
            checkReplica('T%s' % (suffix), 9)
            checkReplica('S%s' % (suffix), 4)