Ejemplo n.º 1
0
    def create_daemon(self, *args, **kwargs):
        self._daemon = DfmsDaemon(*args, **kwargs)

        if 'noNM' not in kwargs or not kwargs['noNM']:
            self.assertTrue(utils.portIsOpen('localhost', constants.NODE_DEFAULT_REST_PORT, _TIMEOUT), 'The NM did not start successfully')
        if 'master' in kwargs and kwargs['master']:
            self.assertTrue(utils.portIsOpen('localhost', constants.MASTER_DEFAULT_REST_PORT, _TIMEOUT), 'The MM did not start successfully')

        self._daemon_t = threading.Thread(target=lambda: self._daemon.start('localhost', 9000))
        self._daemon_t.start()

        # Wait until the daemon's server has started
        # We can't simply check if the port is opened, because the server binds
        # before it is returned to us. In some tests we don't interact with it,
        # and therefore the shutdown of the daemon can occur before the server
        # is even returned to us. This would happen because portIsOpen will
        # succeed with a bound server, even if we haven't serve_forever()'d it
        # yet. In these situations shutting down the daemon will not shut down
        # the http server, and therefore the test will fail when checking that
        # the self._daemon_t is not alive anymore
        #
        # To actually avoid this we need to do some actual HTTP talk, which will
        # ensure the server is actually serving requests, and therefore already
        # in the daemon's hand
        #self.assertTrue(utils.portIsOpen('localhost', 9000, _TIMEOUT))
        try:
            restutils.RestClient('localhost', 9000, 10)._GET('/anything')
        except restutils.RestClientException:
            # We don't care about the result
            pass
Ejemplo n.º 2
0
 def test_nothing_starts(self):
     # Nothing should start now
     self.create_daemon(master=False, noNM=True, disable_zeroconf=True)
     self.assertFalse(
         utils.portIsOpen("localhost", constants.NODE_DEFAULT_REST_PORT, 0), "NM started but it should not have"
     )
     self.assertFalse(
         utils.portIsOpen("localhost", constants.MASTER_DEFAULT_REST_PORT, 0), "NM started but it should not have"
     )
Ejemplo n.º 3
0
    def _request(self, url, method, content=None, headers={}):

        # Do the HTTP stuff...
        if logger.isEnabledFor(logging.DEBUG):
            logger.debug("Sending %s request to %s:%d%s" % (method, self.host, self.port, url))

        if not utils.portIsOpen(self.host, self.port, self.timeout):
            raise RestClientException("Cannot connect to %s:%d after %.2f [s]" % (self.host, self.port, self.timeout))

        self._conn = httplib.HTTPConnection(self.host, self.port)
        self._conn.request(method, url, content, headers)
        self._resp = self._conn.getresponse()

        # Server errors are encoded in the body as json content
        if self._resp.status == httplib.INTERNAL_SERVER_ERROR:
            msg = json.loads(self._resp.read())['err_str']
            if logger.isEnabledFor(logging.WARNING):
                logger.warning('Error found while requesting %s:%d%s: %s' % (self.host, self.port, url, msg))
            raise RestClientException(msg)
        elif self._resp.status != httplib.OK:
            msg = 'Unexpected error while processing %s request for %s:%s%s (status %d): %s' % \
                  (method, self.host, self.port, url, self._resp.status, self._resp.read())
            raise RestClientException(msg)

        return self._resp.read()
Ejemplo n.º 4
0
    def test_start_master_via_rest(self):

        self.create_daemon(master=False, noNM=False, disable_zeroconf=True)

        # Check that the master starts
        self._start('master', httplib.OK)
        self.assertTrue(utils.portIsOpen('localhost', constants.MASTER_DEFAULT_REST_PORT, _TIMEOUT), 'The MM did not start successfully')
Ejemplo n.º 5
0
    def ensureDM(self, host, port=None, timeout=10):

        port = port or self._dmPort

        logger.debug("Checking DM presence at %s:%d", host, port)
        if portIsOpen(host, port, timeout):
            logger.debug("DM already present at %s:%d", host, port)
            return

        # We rely on having ssh keys for this, since we're using
        # the dfms.remote module, which authenticates using public keys
        logger.debug("DM not present at %s:%d, will start it now", host, port)
        self.startDM(host)

        # Wait a bit until the DM starts; if it doesn't we fail
        if not portIsOpen(host, port, timeout):
            raise DaliugeException(
                "DM started at %s:%d, but couldn't connect to it" %
                (host, port))
Ejemplo n.º 6
0
    def test_start_master_via_rest(self):

        self.create_daemon(master=False, noNM=False, disable_zeroconf=True)

        # Check that the master starts
        self._start("master", httplib.OK)
        self.assertTrue(
            utils.portIsOpen("localhost", constants.MASTER_DEFAULT_REST_PORT, _TIMEOUT),
            "The MM did not start successfully",
        )
Ejemplo n.º 7
0
    def ensureDM(self, host, timeout=10):

        # We rely on having ssh keys for this, since we're using
        # the dfms.remote module, which authenticates using public keys
        if logger.isEnabledFor(logging.DEBUG):
                logger.debug("Checking DM presence at %s:%d" % (host, self._dmPort))

        if portIsOpen(host, self._dmPort, timeout):
            if logger.isEnabledFor(logging.DEBUG):
                logger.debug("DM already present at %s:%d" % (host, self._dmPort))
            return

        if logger.isEnabledFor(logging.DEBUG):
                logger.debug("DM not present at %s:%d, will start it now" % (host, self._dmPort))

        self.startDM(host)

        # Wait a bit until the DM starts; if it doesn't we fail
        if not portIsOpen(host, self._dmPort, timeout):
            raise Exception("DM started at %s:%d, but couldn't connect to it" % (host, self._dmPort))
Ejemplo n.º 8
0
    def test_start_dataisland_via_rest(self):

        self.create_daemon(master=True, noNM=False, disable_zeroconf=False)

        # Both managers started fine. If they zeroconf themselves correctly then
        # if we query the MM it should know about its nodes, which should have
        # one element
        nodes = self._get_nodes_from_master(_TIMEOUT)
        self.assertIsNotNone(nodes)
        self.assertEqual(1, len(nodes), "MasterManager didn't find the NodeManager running on the same node")

        # Check that the DataIsland starts with the given nodes
        self._start('dataisland', httplib.OK, {'nodes': nodes})
        self.assertTrue(utils.portIsOpen('localhost', constants.ISLAND_DEFAULT_REST_PORT, _TIMEOUT), 'The DIM did not start successfully')
Ejemplo n.º 9
0
    def create_daemon(self, *args, **kwargs):
        self._daemon = DfmsDaemon(*args, **kwargs)

        if "noNM" not in kwargs or not kwargs["noNM"]:
            self.assertTrue(
                utils.portIsOpen("localhost", constants.NODE_DEFAULT_REST_PORT, _TIMEOUT),
                "The NM did not start successfully",
            )
        if "master" in kwargs and kwargs["master"]:
            self.assertTrue(
                utils.portIsOpen("localhost", constants.MASTER_DEFAULT_REST_PORT, _TIMEOUT),
                "The MM did not start successfully",
            )

        self._daemon_t = threading.Thread(target=lambda: self._daemon.start("localhost", 9000))
        self._daemon_t.start()

        # Wait until the daemon's server has started
        # We can't simply check if the port is opened, because the server binds
        # before it is returned to us. In some tests we don't interact with it,
        # and therefore the shutdown of the daemon can occur before the server
        # is even returned to us. This would happen because portIsOpen will
        # succeed with a bound server, even if we haven't serve_forever()'d it
        # yet. In these situations shutting down the daemon will not shut down
        # the http server, and therefore the test will fail when checking that
        # the self._daemon_t is not alive anymore
        #
        # To actually avoid this we need to do some actual HTTP talk, which will
        # ensure the server is actually serving requests, and therefore already
        # in the daemon's hand
        # self.assertTrue(utils.portIsOpen('localhost', 9000, _TIMEOUT))
        try:
            restutils.RestClient("localhost", 9000, 10)._GET("/anything")
        except restutils.RestClientException:
            # We don't care about the result
            pass
Ejemplo n.º 10
0
    def _request(self, url, method, content=None, headers={}):

        # Do the HTTP stuff...
        logger.debug("Sending %s request to %s:%d%s", method, self.host,
                     self.port, url)

        if not utils.portIsOpen(self.host, self.port, self.timeout):
            raise RestClientException(
                "Cannot connect to %s:%d after %.2f [s]" %
                (self.host, self.port, self.timeout))

        if content and hasattr(content, 'read'):
            headers['Transfer-Encoding'] = 'chunked'
            content = chunked(content)

        self._conn = httplib.HTTPConnection(self.host, self.port)
        self._conn.request(method, url, content, headers)
        self._resp = self._conn.getresponse()

        # Server errors are encoded in the body as json content
        if self._resp.status != httplib.OK:

            msg = 'Error on remote %s@%s:%s%s (status %d): ' % \
                  (method, self.host, self.port, url, self._resp.status)

            try:
                error = json.loads(self._resp.read().decode('utf-8'))
                etype = getattr(exceptions, error['type'])
                eargs = error['args']

                if etype == SubManagerException:
                    for host, args in eargs.items():
                        subetype = getattr(exceptions, args['type'])
                        subargs = args['args']
                        eargs[host] = subetype(*subargs)
                    ex = etype(eargs)
                else:
                    ex = etype(*eargs)
                if hasattr(ex, 'msg'):
                    ex.msg = msg + ex.msg
            except Exception:
                ex = RestClientException(msg + "Unknown")

            raise ex

        if not self._resp.length:
            return None
        return codecs.getreader('utf-8')(self._resp)
Ejemplo n.º 11
0
    def test_start_dataisland_via_rest(self):

        self.create_daemon(master=True, noNM=False, disable_zeroconf=False)

        # Both managers started fine. If they zeroconf themselves correctly then
        # if we query the MM it should know about its nodes, which should have
        # one element
        nodes = self._get_nodes_from_master(_TIMEOUT)
        self.assertIsNotNone(nodes)
        self.assertEquals(1, len(nodes), "MasterManager didn't find the NodeManager running on the same node")

        # Check that the DataIsland starts with the given nodes
        self._start("dataisland", httplib.OK, {"nodes": nodes})
        self.assertTrue(
            utils.portIsOpen("localhost", constants.ISLAND_DEFAULT_REST_PORT, _TIMEOUT),
            "The DIM did not start successfully",
        )
Ejemplo n.º 12
0
def check_host(host, port, timeout=5, check_with_session=False):
    """
    Checks if a given host/port is up and running (i.e., it is open).
    If ``check_with_session`` is ``True`` then it is assumed that the
    host/port combination corresponds to a Node Manager and the check is
    performed by attempting to create and delete a session.
    """
    if not check_with_session:
        return utils.portIsOpen(host, port, timeout)

    try:
        session_id = str(uuid.uuid4())
        with NodeManagerClient(host, port, timeout=timeout) as c:
            c.create_session(session_id)
            c.destroy_session(session_id)
        return True
    except:
        return False
Ejemplo n.º 13
0
def main():

    parser = optparse.OptionParser()
    parser.add_option("-l",
                      "--log_dir",
                      action="store",
                      type="string",
                      dest="log_dir",
                      help="Log directory (required)")
    # if this parameter is present, it means we want to get monitored
    parser.add_option("-m",
                      "--monitor_host",
                      action="store",
                      type="string",
                      dest="monitor_host",
                      help="Monitor host IP (optional)")
    parser.add_option("-o",
                      "--monitor_port",
                      action="store",
                      type="int",
                      dest="monitor_port",
                      help="The port to bind dfms monitor",
                      default=dfms_proxy.default_dfms_monitor_port)
    parser.add_option("-v",
                      "--verbose-level",
                      action="store",
                      type="int",
                      dest="verbose_level",
                      help="Verbosity level (1-3) of the DIM/NM logging",
                      default=1)
    parser.add_option(
        "-z",
        "--zerorun",
        action="store_true",
        dest="zerorun",
        help="Generate a physical graph that takes no time to run",
        default=False)
    parser.add_option(
        "--app",
        action="store",
        type="int",
        dest="app",
        help="The app to use in the PG. 1=SleepApp (default), 2=SleepAndCopy",
        default=0)

    parser.add_option(
        "-t",
        "--max-threads",
        action="store",
        type="int",
        dest="max_threads",
        help=
        "Max thread pool size used for executing drops. 0 (default) means no pool.",
        default=0)

    parser.add_option("-L",
                      "--logical-graph",
                      action="store",
                      type="string",
                      dest="logical_graph",
                      help="The filename of the logical graph to deploy",
                      default=None)
    parser.add_option(
        "-P",
        "--physical-graph",
        action="store",
        type="string",
        dest="physical_graph",
        help="The filename of the physical graph (template) to deploy",
        default=None)

    parser.add_option('-s',
                      '--num_islands',
                      action='store',
                      type='int',
                      dest='num_islands',
                      default=1,
                      help='The number of Data Islands')

    parser.add_option('-d',
                      '--dump',
                      action='store_true',
                      dest='dump',
                      help='dump file base name?',
                      default=False)

    parser.add_option("-c",
                      "--loc",
                      action="store",
                      type="string",
                      dest="loc",
                      help="deployment location (e.g. 'Pawsey' or 'Tianhe2')",
                      default="Pawsey")

    parser.add_option("-u",
                      "--all_nics",
                      action="store_true",
                      dest="all_nics",
                      help="Listen on all NICs for a node manager",
                      default=False)

    parser.add_option('--check-interfaces',
                      action='store_true',
                      dest='check_interfaces',
                      help='Run a small network interfaces test and exit',
                      default=False)
    parser.add_option(
        "-S",
        "--check_with_session",
        action="store_true",
        dest="check_with_session",
        help=
        "Check for node managers' availability by creating/destroy a session",
        default=False)

    (options, _) = parser.parse_args()

    if options.check_interfaces:
        print("From netifaces: %s" % utils.get_local_ip_addr())
        print("From ifconfig: %s" % get_ip())
        sys.exit(0)

    if options.logical_graph and options.physical_graph:
        parser.error(
            "Either a logical graph or physical graph filename must be specified"
        )
    for p in (options.logical_graph, options.physical_graph):
        if p and not os.path.exists(p):
            parser.error("Cannot locate graph file at '{0}'".format(p))

    if (options.monitor_host is not None and options.num_islands > 1):
        parser.error("We do not support proxy monitor multiple islands yet")

    logv = max(min(3, options.verbose_level), 1)

    from mpi4py import MPI  # @UnresolvedImport
    comm = MPI.COMM_WORLD  # @UndefinedVariable
    num_procs = comm.Get_size()
    rank = comm.Get_rank()

    log_dir = "{0}/{1}".format(options.log_dir, rank)
    os.makedirs(log_dir)
    logfile = log_dir + "/start_dfms_cluster.log"
    FORMAT = "%(asctime)-15s [%(levelname)5.5s] [%(threadName)15.15s] %(name)s#%(funcName)s:%(lineno)s %(message)s"
    logging.basicConfig(filename=logfile, level=logging.DEBUG, format=FORMAT)

    if (num_procs > 1 and options.monitor_host is not None):
        logger.info("Trying to start dfms_cluster with proxy")
        run_proxy = True
        threshold = 2
    else:
        logger.info("Trying to start dfms_cluster without proxy")
        run_proxy = False
        threshold = 1

    if (num_procs == threshold):
        logger.warning("No MPI processes left for running Drop Managers")
        run_node_mgr = False
    else:
        run_node_mgr = True

    # attach rank information at the end of IP address for multi-islands
    rank_str = '' if options.num_islands == 1 else ',%s' % rank
    public_ip = get_ip(options.loc)
    ip_adds = '{0}{1}'.format(public_ip, rank_str)
    origin_ip = ip_adds.split(',')[0]
    ip_adds = comm.gather(ip_adds, root=0)

    proxy_ip = None
    if run_proxy:
        # send island/master manager's IP address to the dfms proxy
        # also let island manager know dfms proxy's IP
        if rank == 0:
            mgr_ip = origin_ip
            comm.send(mgr_ip, dest=1)
            proxy_ip = comm.recv(source=1)
        elif rank == 1:
            mgr_ip = comm.recv(source=0)
            proxy_ip = origin_ip
            comm.send(proxy_ip, dest=0)

    set_env(rank)
    if (options.num_islands == 1):
        if (rank != 0):
            if (run_proxy and rank == 1):
                # Wait until the Island Manager is open
                if utils.portIsOpen(mgr_ip, ISLAND_DEFAULT_REST_PORT, 100):
                    start_dfms_proxy(options.loc, mgr_ip,
                                     ISLAND_DEFAULT_REST_PORT,
                                     options.monitor_host,
                                     options.monitor_port)
                else:
                    logger.warning(
                        "Couldn't connect to the main drop manager, proxy not started"
                    )
            elif (run_node_mgr):
                logger.info(
                    "Starting node manager on host {0}".format(origin_ip))
                start_node_mgr(log_dir,
                               logv=logv,
                               max_threads=options.max_threads,
                               host=None if options.all_nics else origin_ip)
        else:

            # 'no_nms' are known not to be NMs
            no_nms = [origin_ip, 'None']
            if proxy_ip:
                no_nms += [proxy_ip]
            node_mgrs = [ip for ip in ip_adds if ip not in no_nms]

            # unroll the graph first (if any) while starting node managers on other nodes
            pgt = None
            if options.logical_graph or options.physical_graph:
                pip_name = utils.fname_to_pipname(options.logical_graph
                                                  or options.physical_graph)
                if options.logical_graph:
                    unrolled = tool.unroll(options.logical_graph, '1',
                                           options.zerorun, apps[options.app])
                    pgt = tool.partition(unrolled, pip_name, len(node_mgrs),
                                         options.num_islands, 'metis')
                    del unrolled
                else:
                    pgt = json.loads(options.physical_graph)

            # Check that which NMs are up and use only those form now on
            node_mgrs = check_hosts(
                node_mgrs,
                NODE_DEFAULT_REST_PORT,
                check_with_session=options.check_with_session,
                timeout=MM_WAIT_TIME)

            # We have a PGT, let's map it and submit it
            if pgt:
                pg = tool.resource_map(pgt, [origin_ip] + node_mgrs, pip_name,
                                       options.num_islands)
                del pgt

                def submit_and_monitor():
                    host, port = 'localhost', ISLAND_DEFAULT_REST_PORT
                    tool.submit(host, port, pg)
                    if options.dump:
                        dump_path = '{0}/monitor'.format(log_dir)
                        monitor_graph(host, port, dump_path)

                threading.Thread(target=submit_and_monitor).start()

            # Start the DIM
            logger.info("Starting island manager on host %s", origin_ip)
            start_dim(node_mgrs, log_dir, logv=logv)

    elif (options.num_islands > 1):
        if (rank == 0):
            # master manager
            # 1. use ip_adds to produce the physical graph
            ip_list = []
            ip_rank_dict = dict()  # k - ip, v - MPI rank
            for ipr in ip_adds:
                iprs = ipr.split(',')
                ip = iprs[0]
                r = iprs[1]
                if (ip == origin_ip or 'None' == ip):
                    continue
                ip_list.append(ip)
                ip_rank_dict[ip] = int(r)

            if (len(ip_list) <= options.num_islands):
                raise Exception(
                    "Insufficient nodes available for node managers")

            # 2 broadcast dim ranks to all nodes to let them know who is the DIM
            dim_ranks = []
            dim_ip_list = ip_list[0:options.num_islands]
            logger.info("A list of DIM IPs: {0}".format(dim_ip_list))
            for dim_ip in dim_ip_list:
                dim_ranks.append(ip_rank_dict[dim_ip])
            dim_ranks = comm.bcast(dim_ranks, root=0)

            # 3 unroll the graph while waiting for node managers to start
            pip_name = utils.fname_to_pipname(options.logical_graph
                                              or options.physical_graph)
            if options.logical_graph:
                unrolled = tool.unroll(options.logical_graph, '1',
                                       options.zerorun, apps[options.app])
                pgt = tool.partition(unrolled, pip_name,
                                     len(ip_list) - 1, options.num_islands,
                                     'metis')
                del unrolled
            else:
                pgt = json.loads(options.physical_graph)

            #logger.info("Waiting all node managers to start in %f seconds", MM_WAIT_TIME)
            node_mgrs = check_hosts(
                ip_list[options.num_islands:],
                NODE_DEFAULT_REST_PORT,
                check_with_session=options.check_with_session,
                timeout=MM_WAIT_TIME)

            # 4.  produce the physical graph based on the available node managers
            # that have already been running (we have to assume island manager
            # will run smoothly in the future)
            logger.info("Master Manager producing the physical graph")
            pg = tool.resource_map(pgt, dim_ip_list + node_mgrs, pip_name,
                                   options.num_islands)

            # 5. parse the pg_spec to get the mapping from islands to node list
            dim_rank_nodes_dict = collections.defaultdict(set)
            for drop in pg:
                dim_ip = drop['island']
                # if (not dim_ip in dim_ip_list):
                #     raise Exception("'{0}' node is not in island list {1}".format(dim_ip, dim_ip_list))
                r = ip_rank_dict[dim_ip]
                n = drop['node']
                dim_rank_nodes_dict[r].add(n)

            # 6 send a node list to each DIM so that it can start
            for dim_ip in dim_ip_list:
                r = ip_rank_dict[dim_ip]
                logger.debug("Sending node list to rank {0}".format(r))
                #TODO this should be in a thread since it is blocking!
                comm.send(list(dim_rank_nodes_dict[r]), dest=r)

            # 7. make sure all DIMs are up running
            dim_ips_up = check_hosts(dim_ip_list,
                                     ISLAND_DEFAULT_REST_PORT,
                                     timeout=MM_WAIT_TIME,
                                     retry=10)
            if len(dim_ips_up) < len(dim_ip_list):
                logger.warning("Not all DIMs were up and running: %d/%d",
                               len(dim_ips_up), len(dim_ip_list))

            # 8. submit the graph in a thread (wait for mm to start)
            def submit():
                if not check_host('localhost',
                                  MASTER_DEFAULT_REST_PORT,
                                  timeout=GRAPH_SUBMIT_WAIT_TIME):
                    logger.warning(
                        "Master Manager didn't come up in %d seconds",
                        GRAPH_SUBMIT_WAIT_TIME)
                tool.submit('localhost', MASTER_DEFAULT_REST_PORT, pg)

            threading.Thread(target=submit).start()

            # 9. start dlgMM using islands IP addresses (this will block)
            start_mm(dim_ip_list, log_dir, logv=logv)

        else:
            dim_ranks = None
            dim_ranks = comm.bcast(dim_ranks, root=0)
            logger.debug("Receiving dim_ranks = {0}, my rank is {1}".format(
                dim_ranks, rank))
            if (rank in dim_ranks):
                logger.debug(
                    "Rank {0} is a DIM preparing for receiving".format(rank))
                # island manager
                # get a list of nodes that are its children from rank 0 (MM)
                nm_list = comm.recv(source=0)
                # no need to wait for node managers since the master manager
                # has already made sure they are up running
                logger.debug("nm_list for DIM {0} is {1}".format(
                    rank, nm_list))
                start_dim(nm_list, log_dir, logv=logv)
            else:
                # node manager
                logger.info(
                    "Starting node manager on host {0}".format(origin_ip))
                start_node_mgr(log_dir,
                               logv=logv,
                               max_threads=options.max_threads,
                               host=None if options.all_nics else origin_ip)
Ejemplo n.º 14
0
 def tearDown(self):
     unittest.TestCase.tearDown(self)
     self._daemon.stop(_TIMEOUT)
     self._daemon_t.join(_TIMEOUT)
     self.assertFalse(self._daemon_t.is_alive(), "Daemon running thread should have finished by now")
     self.assertFalse(utils.portIsOpen("localhost", 9000, 0), "DFMS Daemon REST interface should be off")
Ejemplo n.º 15
0
 def test_nothing_starts(self):
     # Nothing should start now
     self.create_daemon(master=False, noNM=True, disable_zeroconf=True)
     self.assertFalse(utils.portIsOpen('localhost', constants.NODE_DEFAULT_REST_PORT, 0), 'NM started but it should not have')
     self.assertFalse(utils.portIsOpen('localhost', constants.MASTER_DEFAULT_REST_PORT, 0), 'NM started but it should not have')
Ejemplo n.º 16
0
    def test_fullRound(self):
        """
        A test that exercises most of the REST interface exposed on top of the
        NodeManager
        """

        sessionId = 'lala'
        restPort  = 8888

        args = [sys.executable, '-m', 'dfms.manager.cmdline', 'dfmsNM', \
                '--port', str(restPort), '-qqq']
        dmProcess = subprocess.Popen(args)

        with testutils.terminating(dmProcess, 10):

            # Wait until the REST server becomes alive
            self.assertTrue(utils.portIsOpen('localhost', restPort, 10), "REST server didn't come up in time")

            # The DM is still empty
            dmInfo = testutils.get(self, '', restPort)
            self.assertEquals(0, len(dmInfo['sessions']))

            # Create a session and check it exists
            testutils.post(self, '/sessions', restPort, '{"sessionId":"%s"}' % (sessionId))
            dmInfo = testutils.get(self, '', restPort)
            self.assertEquals(1, len(dmInfo['sessions']))
            self.assertEquals(sessionId, dmInfo['sessions'][0]['sessionId'])
            self.assertEquals(SessionStates.PRISTINE, dmInfo['sessions'][0]['status'])

            # Add this complex graph spec to the session
            # The UID of the two leaf nodes of this complex.js graph are T and S
            # PRO-242: use timestamps for final DROPs that get archived into the public NGAS
            graph = json.loads(pkg_resources.resource_string('test', 'graphs/complex.js')) # @UndefinedVariable
            suffix = '_' + str(int(time.time()))
            oidsToReplace = ('S','T')
            for dropSpec in graph:
                if dropSpec['oid'] in oidsToReplace:
                    dropSpec['oid'] += suffix
                for rel in ('inputs','outputs'):
                    if rel in dropSpec:
                        for oid in dropSpec[rel][:]:
                            if oid in oidsToReplace:
                                dropSpec[rel].remove(oid)
                                dropSpec[rel].append(oid + suffix)

            testutils.post(self, '/sessions/%s/graph/append' % (sessionId), restPort, json.dumps(graph))

            # We create two final archiving nodes, but this time from a template
            # available on the server-side
            timeout = 10
            testutils.post(self, '/templates/dfms.manager.repository.archiving_app/materialize?uid=archiving1&host=ngas.ddns.net&port=7777&sessionId=%s&connect_timeout=%f&timeout=%f' % (sessionId, timeout, timeout), restPort)
            testutils.post(self, '/templates/dfms.manager.repository.archiving_app/materialize?uid=archiving2&host=ngas.ddns.net&port=7777&sessionId=%s&connect_timeout=%f&timeout=%f' % (sessionId, timeout, timeout), restPort)

            # And link them to the leaf nodes of the complex graph
            testutils.post(self, '/sessions/%s/graph/link?rhOID=archiving1&lhOID=S%s&linkType=0' % (sessionId, suffix), restPort)
            testutils.post(self, '/sessions/%s/graph/link?rhOID=archiving2&lhOID=T%s&linkType=0' % (sessionId, suffix), restPort)

            # Now we deploy the graph...
            testutils.post(self, '/sessions/%s/deploy' % (sessionId), restPort, 'completed=SL_A,SL_B,SL_C,SL_D,SL_K', mimeType='application/x-www-form-urlencoded')

            # ...and write to all 5 root nodes that are listening in ports
            # starting at 1111
            msg = ''.join([random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in xrange(10)])
            for i in xrange(5):
                self.assertTrue(utils.writeToRemotePort('localhost', 1111+i, msg, 2), "Couldn't write data to localhost:%d" % (1111+i))

            # Wait until the graph has finished its execution. We'll know
            # it finished by polling the status of the session
            while testutils.get(self, '/sessions/%s/status' % (sessionId), restPort) == SessionStates.RUNNING:
                time.sleep(0.2)

            self.assertEquals(SessionStates.FINISHED, testutils.get(self, '/sessions/%s/status' % (sessionId), restPort))
            testutils.delete(self, '/sessions/%s' % (sessionId), restPort)

            # We put an NGAS archiving at the end of the chain, let's check that the DROPs were copied over there
            # Since the graph consists on several SleepAndCopy apps, T should contain the message repeated
            # 9 times, and S should have it 4 times
            def checkReplica(dropId, copies):
                response = ngaslite.retrieve('ngas.ddns.net', dropId)
                buff = response.read()
                self.assertEquals(msg*copies, buff, "%s's replica doesn't look correct" % (dropId))
            checkReplica('T%s' % (suffix), 9)
            checkReplica('S%s' % (suffix), 4)