def create_daemon(self, *args, **kwargs):
self._daemon = DfmsDaemon(*args, **kwargs)
if 'noNM' not in kwargs or not kwargs['noNM']:
self.assertTrue(utils.portIsOpen('localhost', constants.NODE_DEFAULT_REST_PORT, _TIMEOUT), 'The NM did not start successfully')
if 'master' in kwargs and kwargs['master']:
self.assertTrue(utils.portIsOpen('localhost', constants.MASTER_DEFAULT_REST_PORT, _TIMEOUT), 'The MM did not start successfully')
self._daemon_t = threading.Thread(target=lambda: self._daemon.start('localhost', 9000))
self._daemon_t.start()
# Wait until the daemon's server has started
# We can't simply check if the port is opened, because the server binds
# before it is returned to us. In some tests we don't interact with it,
# and therefore the shutdown of the daemon can occur before the server
# is even returned to us. This would happen because portIsOpen will
# succeed with a bound server, even if we haven't serve_forever()'d it
# yet. In these situations shutting down the daemon will not shut down
# the http server, and therefore the test will fail when checking that
# the self._daemon_t is not alive anymore
#
# To actually avoid this we need to do some actual HTTP talk, which will
# ensure the server is actually serving requests, and therefore already
# in the daemon's hand
#self.assertTrue(utils.portIsOpen('localhost', 9000, _TIMEOUT))
try:
restutils.RestClient('localhost', 9000, 10)._GET('/anything')
except restutils.RestClientException:
# We don't care about the result
pass
def test_nothing_starts(self):
# Nothing should start now
self.create_daemon(master=False, noNM=True, disable_zeroconf=True)
self.assertFalse(
utils.portIsOpen("localhost", constants.NODE_DEFAULT_REST_PORT, 0), "NM started but it should not have"
)
self.assertFalse(
utils.portIsOpen("localhost", constants.MASTER_DEFAULT_REST_PORT, 0), "NM started but it should not have"
)
def _request(self, url, method, content=None, headers={}):
# Do the HTTP stuff...
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Sending %s request to %s:%d%s" % (method, self.host, self.port, url))
if not utils.portIsOpen(self.host, self.port, self.timeout):
raise RestClientException("Cannot connect to %s:%d after %.2f [s]" % (self.host, self.port, self.timeout))
self._conn = httplib.HTTPConnection(self.host, self.port)
self._conn.request(method, url, content, headers)
self._resp = self._conn.getresponse()
# Server errors are encoded in the body as json content
if self._resp.status == httplib.INTERNAL_SERVER_ERROR:
msg = json.loads(self._resp.read())['err_str']
if logger.isEnabledFor(logging.WARNING):
logger.warning('Error found while requesting %s:%d%s: %s' % (self.host, self.port, url, msg))
raise RestClientException(msg)
elif self._resp.status != httplib.OK:
msg = 'Unexpected error while processing %s request for %s:%s%s (status %d): %s' % \
(method, self.host, self.port, url, self._resp.status, self._resp.read())
raise RestClientException(msg)
return self._resp.read()
def test_start_master_via_rest(self):
self.create_daemon(master=False, noNM=False, disable_zeroconf=True)
# Check that the master starts
self._start('master', httplib.OK)
self.assertTrue(utils.portIsOpen('localhost', constants.MASTER_DEFAULT_REST_PORT, _TIMEOUT), 'The MM did not start successfully')
def ensureDM(self, host, port=None, timeout=10):
port = port or self._dmPort
logger.debug("Checking DM presence at %s:%d", host, port)
if portIsOpen(host, port, timeout):
logger.debug("DM already present at %s:%d", host, port)
return
# We rely on having ssh keys for this, since we're using
# the dfms.remote module, which authenticates using public keys
logger.debug("DM not present at %s:%d, will start it now", host, port)
self.startDM(host)
# Wait a bit until the DM starts; if it doesn't we fail
if not portIsOpen(host, port, timeout):
raise DaliugeException(
"DM started at %s:%d, but couldn't connect to it" %
(host, port))
def test_start_master_via_rest(self):
self.create_daemon(master=False, noNM=False, disable_zeroconf=True)
# Check that the master starts
self._start("master", httplib.OK)
self.assertTrue(
utils.portIsOpen("localhost", constants.MASTER_DEFAULT_REST_PORT, _TIMEOUT),
"The MM did not start successfully",
)
def ensureDM(self, host, timeout=10):
# We rely on having ssh keys for this, since we're using
# the dfms.remote module, which authenticates using public keys
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Checking DM presence at %s:%d" % (host, self._dmPort))
if portIsOpen(host, self._dmPort, timeout):
if logger.isEnabledFor(logging.DEBUG):
logger.debug("DM already present at %s:%d" % (host, self._dmPort))
return
if logger.isEnabledFor(logging.DEBUG):
logger.debug("DM not present at %s:%d, will start it now" % (host, self._dmPort))
self.startDM(host)
# Wait a bit until the DM starts; if it doesn't we fail
if not portIsOpen(host, self._dmPort, timeout):
raise Exception("DM started at %s:%d, but couldn't connect to it" % (host, self._dmPort))
def test_start_dataisland_via_rest(self):
self.create_daemon(master=True, noNM=False, disable_zeroconf=False)
# Both managers started fine. If they zeroconf themselves correctly then
# if we query the MM it should know about its nodes, which should have
# one element
nodes = self._get_nodes_from_master(_TIMEOUT)
self.assertIsNotNone(nodes)
self.assertEqual(1, len(nodes), "MasterManager didn't find the NodeManager running on the same node")
# Check that the DataIsland starts with the given nodes
self._start('dataisland', httplib.OK, {'nodes': nodes})
self.assertTrue(utils.portIsOpen('localhost', constants.ISLAND_DEFAULT_REST_PORT, _TIMEOUT), 'The DIM did not start successfully')
def create_daemon(self, *args, **kwargs):
self._daemon = DfmsDaemon(*args, **kwargs)
if "noNM" not in kwargs or not kwargs["noNM"]:
self.assertTrue(
utils.portIsOpen("localhost", constants.NODE_DEFAULT_REST_PORT, _TIMEOUT),
"The NM did not start successfully",
)
if "master" in kwargs and kwargs["master"]:
self.assertTrue(
utils.portIsOpen("localhost", constants.MASTER_DEFAULT_REST_PORT, _TIMEOUT),
"The MM did not start successfully",
)
self._daemon_t = threading.Thread(target=lambda: self._daemon.start("localhost", 9000))
self._daemon_t.start()
# Wait until the daemon's server has started
# We can't simply check if the port is opened, because the server binds
# before it is returned to us. In some tests we don't interact with it,
# and therefore the shutdown of the daemon can occur before the server
# is even returned to us. This would happen because portIsOpen will
# succeed with a bound server, even if we haven't serve_forever()'d it
# yet. In these situations shutting down the daemon will not shut down
# the http server, and therefore the test will fail when checking that
# the self._daemon_t is not alive anymore
#
# To actually avoid this we need to do some actual HTTP talk, which will
# ensure the server is actually serving requests, and therefore already
# in the daemon's hand
# self.assertTrue(utils.portIsOpen('localhost', 9000, _TIMEOUT))
try:
restutils.RestClient("localhost", 9000, 10)._GET("/anything")
except restutils.RestClientException:
# We don't care about the result
pass
def _request(self, url, method, content=None, headers={}):
# Do the HTTP stuff...
logger.debug("Sending %s request to %s:%d%s", method, self.host,
self.port, url)
if not utils.portIsOpen(self.host, self.port, self.timeout):
raise RestClientException(
"Cannot connect to %s:%d after %.2f [s]" %
(self.host, self.port, self.timeout))
if content and hasattr(content, 'read'):
headers['Transfer-Encoding'] = 'chunked'
content = chunked(content)
self._conn = httplib.HTTPConnection(self.host, self.port)
self._conn.request(method, url, content, headers)
self._resp = self._conn.getresponse()
# Server errors are encoded in the body as json content
if self._resp.status != httplib.OK:
msg = 'Error on remote %s@%s:%s%s (status %d): ' % \
(method, self.host, self.port, url, self._resp.status)
try:
error = json.loads(self._resp.read().decode('utf-8'))
etype = getattr(exceptions, error['type'])
eargs = error['args']
if etype == SubManagerException:
for host, args in eargs.items():
subetype = getattr(exceptions, args['type'])
subargs = args['args']
eargs[host] = subetype(*subargs)
ex = etype(eargs)
else:
ex = etype(*eargs)
if hasattr(ex, 'msg'):
ex.msg = msg + ex.msg
except Exception:
ex = RestClientException(msg + "Unknown")
raise ex
if not self._resp.length:
return None
return codecs.getreader('utf-8')(self._resp)
def test_start_dataisland_via_rest(self):
self.create_daemon(master=True, noNM=False, disable_zeroconf=False)
# Both managers started fine. If they zeroconf themselves correctly then
# if we query the MM it should know about its nodes, which should have
# one element
nodes = self._get_nodes_from_master(_TIMEOUT)
self.assertIsNotNone(nodes)
self.assertEquals(1, len(nodes), "MasterManager didn't find the NodeManager running on the same node")
# Check that the DataIsland starts with the given nodes
self._start("dataisland", httplib.OK, {"nodes": nodes})
self.assertTrue(
utils.portIsOpen("localhost", constants.ISLAND_DEFAULT_REST_PORT, _TIMEOUT),
"The DIM did not start successfully",
)
def check_host(host, port, timeout=5, check_with_session=False):
"""
Checks if a given host/port is up and running (i.e., it is open).
If ``check_with_session`` is ``True`` then it is assumed that the
host/port combination corresponds to a Node Manager and the check is
performed by attempting to create and delete a session.
"""
if not check_with_session:
return utils.portIsOpen(host, port, timeout)
try:
session_id = str(uuid.uuid4())
with NodeManagerClient(host, port, timeout=timeout) as c:
c.create_session(session_id)
c.destroy_session(session_id)
return True
except:
return False
def main():
parser = optparse.OptionParser()
parser.add_option("-l",
"--log_dir",
action="store",
type="string",
dest="log_dir",
help="Log directory (required)")
# if this parameter is present, it means we want to get monitored
parser.add_option("-m",
"--monitor_host",
action="store",
type="string",
dest="monitor_host",
help="Monitor host IP (optional)")
parser.add_option("-o",
"--monitor_port",
action="store",
type="int",
dest="monitor_port",
help="The port to bind dfms monitor",
default=dfms_proxy.default_dfms_monitor_port)
parser.add_option("-v",
"--verbose-level",
action="store",
type="int",
dest="verbose_level",
help="Verbosity level (1-3) of the DIM/NM logging",
default=1)
parser.add_option(
"-z",
"--zerorun",
action="store_true",
dest="zerorun",
help="Generate a physical graph that takes no time to run",
default=False)
parser.add_option(
"--app",
action="store",
type="int",
dest="app",
help="The app to use in the PG. 1=SleepApp (default), 2=SleepAndCopy",
default=0)
parser.add_option(
"-t",
"--max-threads",
action="store",
type="int",
dest="max_threads",
help=
"Max thread pool size used for executing drops. 0 (default) means no pool.",
default=0)
parser.add_option("-L",
"--logical-graph",
action="store",
type="string",
dest="logical_graph",
help="The filename of the logical graph to deploy",
default=None)
parser.add_option(
"-P",
"--physical-graph",
action="store",
type="string",
dest="physical_graph",
help="The filename of the physical graph (template) to deploy",
default=None)
parser.add_option('-s',
'--num_islands',
action='store',
type='int',
dest='num_islands',
default=1,
help='The number of Data Islands')
parser.add_option('-d',
'--dump',
action='store_true',
dest='dump',
help='dump file base name?',
default=False)
parser.add_option("-c",
"--loc",
action="store",
type="string",
dest="loc",
help="deployment location (e.g. 'Pawsey' or 'Tianhe2')",
default="Pawsey")
parser.add_option("-u",
"--all_nics",
action="store_true",
dest="all_nics",
help="Listen on all NICs for a node manager",
default=False)
parser.add_option('--check-interfaces',
action='store_true',
dest='check_interfaces',
help='Run a small network interfaces test and exit',
default=False)
parser.add_option(
"-S",
"--check_with_session",
action="store_true",
dest="check_with_session",
help=
"Check for node managers' availability by creating/destroy a session",
default=False)
(options, _) = parser.parse_args()
if options.check_interfaces:
print("From netifaces: %s" % utils.get_local_ip_addr())
print("From ifconfig: %s" % get_ip())
sys.exit(0)
if options.logical_graph and options.physical_graph:
parser.error(
"Either a logical graph or physical graph filename must be specified"
)
for p in (options.logical_graph, options.physical_graph):
if p and not os.path.exists(p):
parser.error("Cannot locate graph file at '{0}'".format(p))
if (options.monitor_host is not None and options.num_islands > 1):
parser.error("We do not support proxy monitor multiple islands yet")
logv = max(min(3, options.verbose_level), 1)
from mpi4py import MPI # @UnresolvedImport
comm = MPI.COMM_WORLD # @UndefinedVariable
num_procs = comm.Get_size()
rank = comm.Get_rank()
log_dir = "{0}/{1}".format(options.log_dir, rank)
os.makedirs(log_dir)
logfile = log_dir + "/start_dfms_cluster.log"
FORMAT = "%(asctime)-15s [%(levelname)5.5s] [%(threadName)15.15s] %(name)s#%(funcName)s:%(lineno)s %(message)s"
logging.basicConfig(filename=logfile, level=logging.DEBUG, format=FORMAT)
if (num_procs > 1 and options.monitor_host is not None):
logger.info("Trying to start dfms_cluster with proxy")
run_proxy = True
threshold = 2
else:
logger.info("Trying to start dfms_cluster without proxy")
run_proxy = False
threshold = 1
if (num_procs == threshold):
logger.warning("No MPI processes left for running Drop Managers")
run_node_mgr = False
else:
run_node_mgr = True
# attach rank information at the end of IP address for multi-islands
rank_str = '' if options.num_islands == 1 else ',%s' % rank
public_ip = get_ip(options.loc)
ip_adds = '{0}{1}'.format(public_ip, rank_str)
origin_ip = ip_adds.split(',')[0]
ip_adds = comm.gather(ip_adds, root=0)
proxy_ip = None
if run_proxy:
# send island/master manager's IP address to the dfms proxy
# also let island manager know dfms proxy's IP
if rank == 0:
mgr_ip = origin_ip
comm.send(mgr_ip, dest=1)
proxy_ip = comm.recv(source=1)
elif rank == 1:
mgr_ip = comm.recv(source=0)
proxy_ip = origin_ip
comm.send(proxy_ip, dest=0)
set_env(rank)
if (options.num_islands == 1):
if (rank != 0):
if (run_proxy and rank == 1):
# Wait until the Island Manager is open
if utils.portIsOpen(mgr_ip, ISLAND_DEFAULT_REST_PORT, 100):
start_dfms_proxy(options.loc, mgr_ip,
ISLAND_DEFAULT_REST_PORT,
options.monitor_host,
options.monitor_port)
else:
logger.warning(
"Couldn't connect to the main drop manager, proxy not started"
)
elif (run_node_mgr):
logger.info(
"Starting node manager on host {0}".format(origin_ip))
start_node_mgr(log_dir,
logv=logv,
max_threads=options.max_threads,
host=None if options.all_nics else origin_ip)
else:
# 'no_nms' are known not to be NMs
no_nms = [origin_ip, 'None']
if proxy_ip:
no_nms += [proxy_ip]
node_mgrs = [ip for ip in ip_adds if ip not in no_nms]
# unroll the graph first (if any) while starting node managers on other nodes
pgt = None
if options.logical_graph or options.physical_graph:
pip_name = utils.fname_to_pipname(options.logical_graph
or options.physical_graph)
if options.logical_graph:
unrolled = tool.unroll(options.logical_graph, '1',
options.zerorun, apps[options.app])
pgt = tool.partition(unrolled, pip_name, len(node_mgrs),
options.num_islands, 'metis')
del unrolled
else:
pgt = json.loads(options.physical_graph)
# Check that which NMs are up and use only those form now on
node_mgrs = check_hosts(
node_mgrs,
NODE_DEFAULT_REST_PORT,
check_with_session=options.check_with_session,
timeout=MM_WAIT_TIME)
# We have a PGT, let's map it and submit it
if pgt:
pg = tool.resource_map(pgt, [origin_ip] + node_mgrs, pip_name,
options.num_islands)
del pgt
def submit_and_monitor():
host, port = 'localhost', ISLAND_DEFAULT_REST_PORT
tool.submit(host, port, pg)
if options.dump:
dump_path = '{0}/monitor'.format(log_dir)
monitor_graph(host, port, dump_path)
threading.Thread(target=submit_and_monitor).start()
# Start the DIM
logger.info("Starting island manager on host %s", origin_ip)
start_dim(node_mgrs, log_dir, logv=logv)
elif (options.num_islands > 1):
if (rank == 0):
# master manager
# 1. use ip_adds to produce the physical graph
ip_list = []
ip_rank_dict = dict() # k - ip, v - MPI rank
for ipr in ip_adds:
iprs = ipr.split(',')
ip = iprs[0]
r = iprs[1]
if (ip == origin_ip or 'None' == ip):
continue
ip_list.append(ip)
ip_rank_dict[ip] = int(r)
if (len(ip_list) <= options.num_islands):
raise Exception(
"Insufficient nodes available for node managers")
# 2 broadcast dim ranks to all nodes to let them know who is the DIM
dim_ranks = []
dim_ip_list = ip_list[0:options.num_islands]
logger.info("A list of DIM IPs: {0}".format(dim_ip_list))
for dim_ip in dim_ip_list:
dim_ranks.append(ip_rank_dict[dim_ip])
dim_ranks = comm.bcast(dim_ranks, root=0)
# 3 unroll the graph while waiting for node managers to start
pip_name = utils.fname_to_pipname(options.logical_graph
or options.physical_graph)
if options.logical_graph:
unrolled = tool.unroll(options.logical_graph, '1',
options.zerorun, apps[options.app])
pgt = tool.partition(unrolled, pip_name,
len(ip_list) - 1, options.num_islands,
'metis')
del unrolled
else:
pgt = json.loads(options.physical_graph)
#logger.info("Waiting all node managers to start in %f seconds", MM_WAIT_TIME)
node_mgrs = check_hosts(
ip_list[options.num_islands:],
NODE_DEFAULT_REST_PORT,
check_with_session=options.check_with_session,
timeout=MM_WAIT_TIME)
# 4. produce the physical graph based on the available node managers
# that have already been running (we have to assume island manager
# will run smoothly in the future)
logger.info("Master Manager producing the physical graph")
pg = tool.resource_map(pgt, dim_ip_list + node_mgrs, pip_name,
options.num_islands)
# 5. parse the pg_spec to get the mapping from islands to node list
dim_rank_nodes_dict = collections.defaultdict(set)
for drop in pg:
dim_ip = drop['island']
# if (not dim_ip in dim_ip_list):
# raise Exception("'{0}' node is not in island list {1}".format(dim_ip, dim_ip_list))
r = ip_rank_dict[dim_ip]
n = drop['node']
dim_rank_nodes_dict[r].add(n)
# 6 send a node list to each DIM so that it can start
for dim_ip in dim_ip_list:
r = ip_rank_dict[dim_ip]
logger.debug("Sending node list to rank {0}".format(r))
#TODO this should be in a thread since it is blocking!
comm.send(list(dim_rank_nodes_dict[r]), dest=r)
# 7. make sure all DIMs are up running
dim_ips_up = check_hosts(dim_ip_list,
ISLAND_DEFAULT_REST_PORT,
timeout=MM_WAIT_TIME,
retry=10)
if len(dim_ips_up) < len(dim_ip_list):
logger.warning("Not all DIMs were up and running: %d/%d",
len(dim_ips_up), len(dim_ip_list))
# 8. submit the graph in a thread (wait for mm to start)
def submit():
if not check_host('localhost',
MASTER_DEFAULT_REST_PORT,
timeout=GRAPH_SUBMIT_WAIT_TIME):
logger.warning(
"Master Manager didn't come up in %d seconds",
GRAPH_SUBMIT_WAIT_TIME)
tool.submit('localhost', MASTER_DEFAULT_REST_PORT, pg)
threading.Thread(target=submit).start()
# 9. start dlgMM using islands IP addresses (this will block)
start_mm(dim_ip_list, log_dir, logv=logv)
else:
dim_ranks = None
dim_ranks = comm.bcast(dim_ranks, root=0)
logger.debug("Receiving dim_ranks = {0}, my rank is {1}".format(
dim_ranks, rank))
if (rank in dim_ranks):
logger.debug(
"Rank {0} is a DIM preparing for receiving".format(rank))
# island manager
# get a list of nodes that are its children from rank 0 (MM)
nm_list = comm.recv(source=0)
# no need to wait for node managers since the master manager
# has already made sure they are up running
logger.debug("nm_list for DIM {0} is {1}".format(
rank, nm_list))
start_dim(nm_list, log_dir, logv=logv)
else:
# node manager
logger.info(
"Starting node manager on host {0}".format(origin_ip))
start_node_mgr(log_dir,
logv=logv,
max_threads=options.max_threads,
host=None if options.all_nics else origin_ip)
def tearDown(self):
unittest.TestCase.tearDown(self)
self._daemon.stop(_TIMEOUT)
self._daemon_t.join(_TIMEOUT)
self.assertFalse(self._daemon_t.is_alive(), "Daemon running thread should have finished by now")
self.assertFalse(utils.portIsOpen("localhost", 9000, 0), "DFMS Daemon REST interface should be off")
def test_nothing_starts(self):
# Nothing should start now
self.create_daemon(master=False, noNM=True, disable_zeroconf=True)
self.assertFalse(utils.portIsOpen('localhost', constants.NODE_DEFAULT_REST_PORT, 0), 'NM started but it should not have')
self.assertFalse(utils.portIsOpen('localhost', constants.MASTER_DEFAULT_REST_PORT, 0), 'NM started but it should not have')
def test_fullRound(self):
"""
A test that exercises most of the REST interface exposed on top of the
NodeManager
"""
sessionId = 'lala'
restPort = 8888
args = [sys.executable, '-m', 'dfms.manager.cmdline', 'dfmsNM', \
'--port', str(restPort), '-qqq']
dmProcess = subprocess.Popen(args)
with testutils.terminating(dmProcess, 10):
# Wait until the REST server becomes alive
self.assertTrue(utils.portIsOpen('localhost', restPort, 10), "REST server didn't come up in time")
# The DM is still empty
dmInfo = testutils.get(self, '', restPort)
self.assertEquals(0, len(dmInfo['sessions']))
# Create a session and check it exists
testutils.post(self, '/sessions', restPort, '{"sessionId":"%s"}' % (sessionId))
dmInfo = testutils.get(self, '', restPort)
self.assertEquals(1, len(dmInfo['sessions']))
self.assertEquals(sessionId, dmInfo['sessions'][0]['sessionId'])
self.assertEquals(SessionStates.PRISTINE, dmInfo['sessions'][0]['status'])
# Add this complex graph spec to the session
# The UID of the two leaf nodes of this complex.js graph are T and S
# PRO-242: use timestamps for final DROPs that get archived into the public NGAS
graph = json.loads(pkg_resources.resource_string('test', 'graphs/complex.js')) # @UndefinedVariable
suffix = '_' + str(int(time.time()))
oidsToReplace = ('S','T')
for dropSpec in graph:
if dropSpec['oid'] in oidsToReplace:
dropSpec['oid'] += suffix
for rel in ('inputs','outputs'):
if rel in dropSpec:
for oid in dropSpec[rel][:]:
if oid in oidsToReplace:
dropSpec[rel].remove(oid)
dropSpec[rel].append(oid + suffix)
testutils.post(self, '/sessions/%s/graph/append' % (sessionId), restPort, json.dumps(graph))
# We create two final archiving nodes, but this time from a template
# available on the server-side
timeout = 10
testutils.post(self, '/templates/dfms.manager.repository.archiving_app/materialize?uid=archiving1&host=ngas.ddns.net&port=7777&sessionId=%s&connect_timeout=%f&timeout=%f' % (sessionId, timeout, timeout), restPort)
testutils.post(self, '/templates/dfms.manager.repository.archiving_app/materialize?uid=archiving2&host=ngas.ddns.net&port=7777&sessionId=%s&connect_timeout=%f&timeout=%f' % (sessionId, timeout, timeout), restPort)
# And link them to the leaf nodes of the complex graph
testutils.post(self, '/sessions/%s/graph/link?rhOID=archiving1&lhOID=S%s&linkType=0' % (sessionId, suffix), restPort)
testutils.post(self, '/sessions/%s/graph/link?rhOID=archiving2&lhOID=T%s&linkType=0' % (sessionId, suffix), restPort)
# Now we deploy the graph...
testutils.post(self, '/sessions/%s/deploy' % (sessionId), restPort, 'completed=SL_A,SL_B,SL_C,SL_D,SL_K', mimeType='application/x-www-form-urlencoded')
# ...and write to all 5 root nodes that are listening in ports
# starting at 1111
msg = ''.join([random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in xrange(10)])
for i in xrange(5):
self.assertTrue(utils.writeToRemotePort('localhost', 1111+i, msg, 2), "Couldn't write data to localhost:%d" % (1111+i))
# Wait until the graph has finished its execution. We'll know
# it finished by polling the status of the session
while testutils.get(self, '/sessions/%s/status' % (sessionId), restPort) == SessionStates.RUNNING:
time.sleep(0.2)
self.assertEquals(SessionStates.FINISHED, testutils.get(self, '/sessions/%s/status' % (sessionId), restPort))
testutils.delete(self, '/sessions/%s' % (sessionId), restPort)
# We put an NGAS archiving at the end of the chain, let's check that the DROPs were copied over there
# Since the graph consists on several SleepAndCopy apps, T should contain the message repeated
# 9 times, and S should have it 4 times
def checkReplica(dropId, copies):
response = ngaslite.retrieve('ngas.ddns.net', dropId)
buff = response.read()
self.assertEquals(msg*copies, buff, "%s's replica doesn't look correct" % (dropId))
checkReplica('T%s' % (suffix), 9)
checkReplica('S%s' % (suffix), 4)