def test_pipeline(self):
"""A pipeline from an LG all the way to a finished graph execution"""
lg = pkg_resources.resource_filename( # @UndefinedVariable
"test.dropmake", "logical_graphs/cont_img.graph")
fill = tool.start_process("fill", ["-L", lg], stdout=subprocess.PIPE)
unroll = tool.start_process("unroll", ["-z", "--app", "1"],
stdin=fill.stdout,
stdout=subprocess.PIPE)
partition = tool.start_process("partition",
stdin=unroll.stdout,
stdout=subprocess.PIPE)
map_ = tool.start_process(
"map",
["-N", "127.0.0.1,127.0.0.1"],
stdin=partition.stdout,
stdout=subprocess.PIPE,
)
mapped_graph, _ = map_.communicate()
for proc in fill, unroll, partition, map_:
self.assertEqual(proc.wait(), 0)
# It's valid JSON content, and actually a physical graph
mapped_graph = json.loads(common.b2s(mapped_graph))
# TODO: REPRODATA ATTACHED
mapped_graph.pop() # Get rid of reprodata
self.assertTrue(list(common.get_roots(mapped_graph)))
self.assertTrue(list(common.get_leaves(mapped_graph)))
def _run_graph(self, graph, completed_uids, timeout=5):
sessionId = "lala"
restPort = 8989
args = ["--port", str(restPort), "-N", hostname, "-qq"]
logger.debug("Starting NM on port %d", restPort)
c = client.NodeManagerClient(port=restPort)
dimProcess = tool.start_process("dim", args)
with testutils.terminating(dimProcess, timeout=timeout):
c.create_session(sessionId)
logger.info("Appending graph")
c.append_graph(sessionId, graph)
# What we are actually trying to measure with all this stuff
start = time.time()
c.deploy_session(sessionId, completed_uids)
delta = time.time() - start
# A minute is more than enough, in my PC it takes around 4 or 5 [s]
# A minute is also way less than the ~2 [h] we observed in AWS
self.assertLessEqual(
delta, 60, "It took way too much time to create all drops"
)
def test_pipeline(self):
"""A pipeline from an LG all the way to a finished graph execution"""
lg = pkg_resources.resource_filename( # @UndefinedVariable
'test.dropmake', 'logical_graphs/cont_img.json')
fill = tool.start_process('fill', ['-L', lg], stdout=subprocess.PIPE)
unroll = tool.start_process('unroll', ['-z', '--app', '1'], stdin=fill.stdout, stdout=subprocess.PIPE)
partition = tool.start_process('partition', stdin=unroll.stdout, stdout=subprocess.PIPE)
map_ = tool.start_process('map', ['-N', '127.0.0.1,127.0.0.1'], stdin=partition.stdout, stdout=subprocess.PIPE)
mapped_graph, _ = map_.communicate()
for proc in fill, unroll, partition, map_:
self.assertEqual(proc.wait(), 0)
# It's valid JSON content, and actually a physical graph
mapped_graph = json.loads(common.b2s(mapped_graph))
self.assertTrue(list(common.get_roots(mapped_graph)))
self.assertTrue(list(common.get_leaves(mapped_graph)))
def test_cmdhelp(self):
"""Checks that all dlg commands have a help"""
for cmd in tool.commands:
p = tool.start_process(cmd, ['-h'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = p.communicate()
common.wait_or_kill(p, timeout=10)
self.assertEqual(
0, p.returncode,
'cmd: %s, out: %s' % (cmd + ' -h', common.b2s(out + err)))
def setUp(self):
unittest.TestCase.setUp(self)
self.temp_dir = tempfile.mkdtemp()
args = [
'-d', lg_dir, '-t', self.temp_dir, '-p',
str(lgweb_port), '-H', 'localhost'
]
self.devnull = open(os.devnull, 'wb')
self.web_proc = tool.start_process('lgweb',
args,
stdout=self.devnull,
stderr=self.devnull)
def setUp(self):
unittest.TestCase.setUp(self)
self.temp_dir = tempfile.mkdtemp()
args = [
"-d",
lg_dir,
"-t",
self.temp_dir,
"-p",
str(lgweb_port),
"-H",
"localhost",
]
self.devnull = open(os.devnull, "wb")
self.web_proc = tool.start_process(
"lgweb", args, stdout=self.devnull, stderr=self.devnull
)
def start_node_mgr(log_dir,
my_ip,
logv=1,
max_threads=0,
host=None,
event_listeners="",
use_tool=True):
"""
Start node manager
"""
LOGGER.info("Starting node manager on host %s", my_ip)
host = host or "0.0.0.0"
log_level = "v" * logv
args = [
"-l",
log_dir,
"-w",
get_workspace_dir(log_dir),
"-%s" % log_level,
"-H",
host,
"-m",
"1024",
"-t",
str(max_threads),
"--no-dlm",
]
if event_listeners:
args += ["--event-listeners", event_listeners]
if use_tool:
# This returns immediately
proc = tool.start_process("nm", args)
LOGGER.info("Node manager process started with pid %d", proc.pid)
return proc
else:
# This blocks until NM shutdown externally
return cmdline.dlgNM(optparse.OptionParser(), args)
def start_dim(node_list, log_dir, origin_ip, logv=1):
"""
Start data island manager
"""
LOGGER.info("Starting island manager on host %s for node managers %r",
origin_ip, node_list)
log_level = "v" * logv
args = [
"-l",
log_dir,
"-w",
get_workspace_dir(log_dir),
"-%s" % log_level,
"-N",
",".join(node_list),
"-H",
"0.0.0.0",
"-m",
"2048",
]
proc = tool.start_process("dim", args)
LOGGER.info("Island manager process started with pid %d", proc.pid)
return proc
def start_mm(node_list, log_dir, logv=1):
"""
Start master manager
node_list: a list of node address that host DIMs
"""
log_level = "v" * logv
parser = optparse.OptionParser()
args = [
"-l",
log_dir,
"-w",
get_workspace_dir(log_dir),
"-N",
",".join(node_list),
"-%s" % log_level,
"-H",
"0.0.0.0",
"-m",
"2048",
]
proc = tool.start_process("mm", args)
LOGGER.info("Master manager process started with pid %d", proc.pid)
return proc
def setUp(self):
unittest.TestCase.setUp(self)
env = os.environ.copy()
env['PYTHONPATH'] = env.get('PYTHONPATH', '') + ":" + os.getcwd()
self.dmProcess = tool.start_process('nm', env=env)
def setUp(self):
unittest.TestCase.setUp(self)
args = ['-H', hostname, '-qq']
self.dmProcess = tool.start_process('nm', args)
def test_fullRound(self):
"""
A test that exercises most of the REST interface exposed on top of the
DataIslandManager
"""
sessionId = 'lala'
restPort = 8888
args = ['--port', str(restPort), '-N', hostname, '-qqq']
dimProcess = tool.start_process('dim', args)
with testutils.terminating(dimProcess, 10):
# Wait until the REST server becomes alive
self.assertTrue(utils.portIsOpen('localhost', restPort, 10),
"REST server didn't come up in time")
# The DIM is still empty
sessions = testutils.get(self, '/sessions', restPort)
self.assertEqual(0, len(sessions))
dimStatus = testutils.get(self, '', restPort)
self.assertEqual(1, len(dimStatus['hosts']))
self.assertEqual(hostname, dimStatus['hosts'][0])
self.assertEqual(0, len(dimStatus['sessionIds']))
# Create a session and check it exists
testutils.post(self, '/sessions', restPort,
'{"sessionId":"%s"}' % (sessionId))
sessions = testutils.get(self, '/sessions', restPort)
self.assertEqual(1, len(sessions))
self.assertEqual(sessionId, sessions[0]['sessionId'])
self.assertDictEqual({hostname: SessionStates.PRISTINE},
sessions[0]['status'])
# Add this complex graph spec to the session
# The UID of the two leaf nodes of this complex.js graph are T and S
# Since the original complexGraph doesn't have node information
# we need to add it manually before submitting -- otherwise it will
# get rejected by the DIM.
with pkg_resources.resource_stream(
'test', 'graphs/complex.js') as f: # @UndefinedVariable
complexGraphSpec = json.load(codecs.getreader('utf-8')(f))
for dropSpec in complexGraphSpec:
dropSpec['node'] = hostname
testutils.post(self, '/sessions/%s/graph/append' % (sessionId),
restPort, json.dumps(complexGraphSpec))
self.assertEqual({hostname: SessionStates.BUILDING},
testutils.get(self,
'/sessions/%s/status' % (sessionId),
restPort))
# Now we deploy the graph...
testutils.post(self,
'/sessions/%s/deploy' % (sessionId),
restPort,
"completed=SL_A,SL_B,SL_C,SL_D,SL_K",
mimeType='application/x-www-form-urlencoded')
self.assertEqual({hostname: SessionStates.RUNNING},
testutils.get(self,
'/sessions/%s/status' % (sessionId),
restPort))
# ...and write to all 5 root nodes that are listening in ports
# starting at 1111
msg = os.urandom(10)
for i in range(5):
utils.write_to(
'localhost', 1111 + i, msg,
2), "Couldn't write data to localhost:%d" % (1111 + i)
# Wait until the graph has finished its execution. We'll know
# it finished by polling the status of the session
while SessionStates.RUNNING in testutils.get(
self, '/sessions/%s/status' % (sessionId),
restPort).values():
time.sleep(0.2)
self.assertEqual({hostname: SessionStates.FINISHED},
testutils.get(self,
'/sessions/%s/status' % (sessionId),
restPort))
testutils.delete(self, '/sessions/%s' % (sessionId), restPort)
sessions = testutils.get(self, '/sessions', restPort)
self.assertEqual(0, len(sessions))
def test_fullRound(self):
"""
A test that exercises most of the REST interface exposed on top of the
DataIslandManager
"""
sessionId = "lala"
restPort = 8989 # don't interfere with EAGLE default port
args = ["--port", str(restPort), "-N", hostname, "-qqq"]
dimProcess = tool.start_process("dim", args)
with testutils.terminating(dimProcess, timeout=10):
# Wait until the REST server becomes alive
self.assertTrue(
utils.portIsOpen("localhost", restPort, timeout=10),
"REST server didn't come up in time",
)
# The DIM is still empty
sessions = testutils.get(self, "/sessions", restPort)
self.assertEqual(0, len(sessions))
dimStatus = testutils.get(self, "", restPort)
self.assertEqual(1, len(dimStatus["hosts"]))
self.assertEqual(hostname, dimStatus["hosts"][0])
self.assertEqual(0, len(dimStatus["sessionIds"]))
# Create a session and check it exists
testutils.post(
self, "/sessions", restPort, '{"sessionId":"%s"}' % (sessionId)
)
sessions = testutils.get(self, "/sessions", restPort)
self.assertEqual(1, len(sessions))
self.assertEqual(sessionId, sessions[0]["sessionId"])
self.assertDictEqual(
{hostname: SessionStates.PRISTINE}, sessions[0]["status"]
)
# Add this complex graph spec to the session
# The UID of the two leaf nodes of this complex.js graph are T and S
# Since the original complexGraph doesn't have node information
# we need to add it manually before submitting -- otherwise it will
# get rejected by the DIM.
with pkg_resources.resource_stream(
"test", "graphs/complex.js"
) as f: # @UndefinedVariable
complexGraphSpec = json.load(codecs.getreader("utf-8")(f))
logger.debug(f"Loaded graph: {f}")
for dropSpec in complexGraphSpec:
dropSpec["node"] = hostname
testutils.post(
self,
"/sessions/%s/graph/append" % (sessionId),
restPort,
json.dumps(complexGraphSpec),
)
self.assertEqual(
{hostname: SessionStates.BUILDING},
testutils.get(self, "/sessions/%s/status" % (sessionId), restPort),
)
# Now we deploy the graph...
testutils.post(
self,
"/sessions/%s/deploy" % (sessionId),
restPort,
"completed=SL_A,SL_B,SL_C,SL_D,SL_K",
mimeType="application/x-www-form-urlencoded",
)
self.assertEqual(
{hostname: SessionStates.RUNNING},
testutils.get(self, "/sessions/%s/status" % (sessionId), restPort),
)
# ...and write to all 5 root nodes that are listening in ports
# starting at 1111
msg = os.urandom(10)
for i in range(5):
utils.write_to(
"localhost", 1111 + i, msg, 2
), "Couldn't write data to localhost:%d" % (1111 + i)
# Wait until the graph has finished its execution. We'll know
# it finished by polling the status of the session
while (
SessionStates.RUNNING
in testutils.get(
self, "/sessions/%s/status" % (sessionId), restPort
).values()
):
time.sleep(0.2)
self.assertEqual(
{hostname: SessionStates.FINISHED},
testutils.get(self, "/sessions/%s/status" % (sessionId), restPort),
)
testutils.delete(self, "/sessions/%s" % (sessionId), restPort)
sessions = testutils.get(self, "/sessions", restPort)
self.assertEqual(0, len(sessions))
def setUp(self):
unittest.TestCase.setUp(self)
env = os.environ.copy()
env["PYTHONPATH"] = (env.get("PYTHONPATH", "") + ":" + os.getcwd() +
"/daliuge-engine")
self.dmProcess = tool.start_process("nm", ["-vvv"], env=env)
def setUp(self):
unittest.TestCase.setUp(self)
args = ["-H", hostname, "-qq"]
self.dmProcess = tool.start_process("nm", args)
