def _run_graph(self, graph, completed_uids, timeout=5):
sessionId = "lala"
restPort = 8989
args = ["--port", str(restPort), "-N", hostname, "-qq"]
logger.debug("Starting NM on port %d", restPort)
c = client.NodeManagerClient(port=restPort)
dimProcess = tool.start_process("dim", args)
with testutils.terminating(dimProcess, timeout=timeout):
c.create_session(sessionId)
logger.info("Appending graph")
c.append_graph(sessionId, graph)
# What we are actually trying to measure with all this stuff
start = time.time()
c.deploy_session(sessionId, completed_uids)
delta = time.time() - start
# A minute is more than enough, in my PC it takes around 4 or 5 [s]
# A minute is also way less than the ~2 [h] we observed in AWS
self.assertLessEqual(
delta, 60, "It took way too much time to create all drops"
)
def _run_graph(self, graph, completed_uids, timeout):
sessionId = 'lala'
restPort = 8888
args = [sys.executable, '-m', 'dfms.manager.cmdline', 'dfmsDIM', \
'--port', str(restPort), '-N', hostname, '-qq']
c = client.NodeManagerClient(port=restPort)
dimProcess = subprocess.Popen(args)
with testutils.terminating(dimProcess, timeout):
c.create_session(sessionId)
logger.info("Appending graph")
c.append_graph(sessionId, graph)
# What we are actually trying to measure with all this stuff
start = time.time()
c.deploy_session(sessionId, completed_uids)
delta = time.time() - start
# A minute is more than enough, in my PC it takes around 4 or 5 [s]
# A minute is also way less than the ~2 [h] we observed in AWS
self.assertLessEqual(
delta, 60, "It took way too much time to create all drops")
def test_fullRound(self):
"""
A test that exercises most of the REST interface exposed on top of the
DataIslandManager
"""
sessionId = 'lala'
restPort = 8888
args = ['--port', str(restPort), '-N', hostname, '-qqq']
dimProcess = tool.start_process('dim', args)
with testutils.terminating(dimProcess, 10):
# Wait until the REST server becomes alive
self.assertTrue(utils.portIsOpen('localhost', restPort, 10),
"REST server didn't come up in time")
# The DIM is still empty
sessions = testutils.get(self, '/sessions', restPort)
self.assertEqual(0, len(sessions))
dimStatus = testutils.get(self, '', restPort)
self.assertEqual(1, len(dimStatus['hosts']))
self.assertEqual(hostname, dimStatus['hosts'][0])
self.assertEqual(0, len(dimStatus['sessionIds']))
# Create a session and check it exists
testutils.post(self, '/sessions', restPort,
'{"sessionId":"%s"}' % (sessionId))
sessions = testutils.get(self, '/sessions', restPort)
self.assertEqual(1, len(sessions))
self.assertEqual(sessionId, sessions[0]['sessionId'])
self.assertDictEqual({hostname: SessionStates.PRISTINE},
sessions[0]['status'])
# Add this complex graph spec to the session
# The UID of the two leaf nodes of this complex.js graph are T and S
# Since the original complexGraph doesn't have node information
# we need to add it manually before submitting -- otherwise it will
# get rejected by the DIM.
with pkg_resources.resource_stream(
'test', 'graphs/complex.js') as f: # @UndefinedVariable
complexGraphSpec = json.load(codecs.getreader('utf-8')(f))
for dropSpec in complexGraphSpec:
dropSpec['node'] = hostname
testutils.post(self, '/sessions/%s/graph/append' % (sessionId),
restPort, json.dumps(complexGraphSpec))
self.assertEqual({hostname: SessionStates.BUILDING},
testutils.get(self,
'/sessions/%s/status' % (sessionId),
restPort))
# Now we deploy the graph...
testutils.post(self,
'/sessions/%s/deploy' % (sessionId),
restPort,
"completed=SL_A,SL_B,SL_C,SL_D,SL_K",
mimeType='application/x-www-form-urlencoded')
self.assertEqual({hostname: SessionStates.RUNNING},
testutils.get(self,
'/sessions/%s/status' % (sessionId),
restPort))
# ...and write to all 5 root nodes that are listening in ports
# starting at 1111
msg = os.urandom(10)
for i in range(5):
utils.write_to(
'localhost', 1111 + i, msg,
2), "Couldn't write data to localhost:%d" % (1111 + i)
# Wait until the graph has finished its execution. We'll know
# it finished by polling the status of the session
while SessionStates.RUNNING in testutils.get(
self, '/sessions/%s/status' % (sessionId),
restPort).values():
time.sleep(0.2)
self.assertEqual({hostname: SessionStates.FINISHED},
testutils.get(self,
'/sessions/%s/status' % (sessionId),
restPort))
testutils.delete(self, '/sessions/%s' % (sessionId), restPort)
sessions = testutils.get(self, '/sessions', restPort)
self.assertEqual(0, len(sessions))
def test_fullRound(self):
"""
A test that exercises most of the REST interface exposed on top of the
DataIslandManager
"""
sessionId = "lala"
restPort = 8989 # don't interfere with EAGLE default port
args = ["--port", str(restPort), "-N", hostname, "-qqq"]
dimProcess = tool.start_process("dim", args)
with testutils.terminating(dimProcess, timeout=10):
# Wait until the REST server becomes alive
self.assertTrue(
utils.portIsOpen("localhost", restPort, timeout=10),
"REST server didn't come up in time",
)
# The DIM is still empty
sessions = testutils.get(self, "/sessions", restPort)
self.assertEqual(0, len(sessions))
dimStatus = testutils.get(self, "", restPort)
self.assertEqual(1, len(dimStatus["hosts"]))
self.assertEqual(hostname, dimStatus["hosts"][0])
self.assertEqual(0, len(dimStatus["sessionIds"]))
# Create a session and check it exists
testutils.post(
self, "/sessions", restPort, '{"sessionId":"%s"}' % (sessionId)
)
sessions = testutils.get(self, "/sessions", restPort)
self.assertEqual(1, len(sessions))
self.assertEqual(sessionId, sessions[0]["sessionId"])
self.assertDictEqual(
{hostname: SessionStates.PRISTINE}, sessions[0]["status"]
)
# Add this complex graph spec to the session
# The UID of the two leaf nodes of this complex.js graph are T and S
# Since the original complexGraph doesn't have node information
# we need to add it manually before submitting -- otherwise it will
# get rejected by the DIM.
with pkg_resources.resource_stream(
"test", "graphs/complex.js"
) as f: # @UndefinedVariable
complexGraphSpec = json.load(codecs.getreader("utf-8")(f))
logger.debug(f"Loaded graph: {f}")
for dropSpec in complexGraphSpec:
dropSpec["node"] = hostname
testutils.post(
self,
"/sessions/%s/graph/append" % (sessionId),
restPort,
json.dumps(complexGraphSpec),
)
self.assertEqual(
{hostname: SessionStates.BUILDING},
testutils.get(self, "/sessions/%s/status" % (sessionId), restPort),
)
# Now we deploy the graph...
testutils.post(
self,
"/sessions/%s/deploy" % (sessionId),
restPort,
"completed=SL_A,SL_B,SL_C,SL_D,SL_K",
mimeType="application/x-www-form-urlencoded",
)
self.assertEqual(
{hostname: SessionStates.RUNNING},
testutils.get(self, "/sessions/%s/status" % (sessionId), restPort),
)
# ...and write to all 5 root nodes that are listening in ports
# starting at 1111
msg = os.urandom(10)
for i in range(5):
utils.write_to(
"localhost", 1111 + i, msg, 2
), "Couldn't write data to localhost:%d" % (1111 + i)
# Wait until the graph has finished its execution. We'll know
# it finished by polling the status of the session
while (
SessionStates.RUNNING
in testutils.get(
self, "/sessions/%s/status" % (sessionId), restPort
).values()
):
time.sleep(0.2)
self.assertEqual(
{hostname: SessionStates.FINISHED},
testutils.get(self, "/sessions/%s/status" % (sessionId), restPort),
)
testutils.delete(self, "/sessions/%s" % (sessionId), restPort)
sessions = testutils.get(self, "/sessions", restPort)
self.assertEqual(0, len(sessions))
def test_fullRound(self):
"""
A test that exercises most of the REST interface exposed on top of the
NodeManager
"""
sessionId = 'lala'
restPort = 8888
args = [sys.executable, '-m', 'dfms.manager.cmdline', 'dfmsNM', \
'--port', str(restPort), '-qqq']
dmProcess = subprocess.Popen(args)
with testutils.terminating(dmProcess, 10):
# Wait until the REST server becomes alive
self.assertTrue(utils.portIsOpen('localhost', restPort, 10), "REST server didn't come up in time")
# The DM is still empty
dmInfo = testutils.get(self, '', restPort)
self.assertEquals(0, len(dmInfo['sessions']))
# Create a session and check it exists
testutils.post(self, '/sessions', restPort, '{"sessionId":"%s"}' % (sessionId))
dmInfo = testutils.get(self, '', restPort)
self.assertEquals(1, len(dmInfo['sessions']))
self.assertEquals(sessionId, dmInfo['sessions'][0]['sessionId'])
self.assertEquals(SessionStates.PRISTINE, dmInfo['sessions'][0]['status'])
# Add this complex graph spec to the session
# The UID of the two leaf nodes of this complex.js graph are T and S
# PRO-242: use timestamps for final DROPs that get archived into the public NGAS
graph = json.loads(pkg_resources.resource_string('test', 'graphs/complex.js')) # @UndefinedVariable
suffix = '_' + str(int(time.time()))
oidsToReplace = ('S','T')
for dropSpec in graph:
if dropSpec['oid'] in oidsToReplace:
dropSpec['oid'] += suffix
for rel in ('inputs','outputs'):
if rel in dropSpec:
for oid in dropSpec[rel][:]:
if oid in oidsToReplace:
dropSpec[rel].remove(oid)
dropSpec[rel].append(oid + suffix)
testutils.post(self, '/sessions/%s/graph/append' % (sessionId), restPort, json.dumps(graph))
# We create two final archiving nodes, but this time from a template
# available on the server-side
timeout = 10
testutils.post(self, '/templates/dfms.manager.repository.archiving_app/materialize?uid=archiving1&host=ngas.ddns.net&port=7777&sessionId=%s&connect_timeout=%f&timeout=%f' % (sessionId, timeout, timeout), restPort)
testutils.post(self, '/templates/dfms.manager.repository.archiving_app/materialize?uid=archiving2&host=ngas.ddns.net&port=7777&sessionId=%s&connect_timeout=%f&timeout=%f' % (sessionId, timeout, timeout), restPort)
# And link them to the leaf nodes of the complex graph
testutils.post(self, '/sessions/%s/graph/link?rhOID=archiving1&lhOID=S%s&linkType=0' % (sessionId, suffix), restPort)
testutils.post(self, '/sessions/%s/graph/link?rhOID=archiving2&lhOID=T%s&linkType=0' % (sessionId, suffix), restPort)
# Now we deploy the graph...
testutils.post(self, '/sessions/%s/deploy' % (sessionId), restPort, 'completed=SL_A,SL_B,SL_C,SL_D,SL_K', mimeType='application/x-www-form-urlencoded')
# ...and write to all 5 root nodes that are listening in ports
# starting at 1111
msg = ''.join([random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in xrange(10)])
for i in xrange(5):
self.assertTrue(utils.writeToRemotePort('localhost', 1111+i, msg, 2), "Couldn't write data to localhost:%d" % (1111+i))
# Wait until the graph has finished its execution. We'll know
# it finished by polling the status of the session
while testutils.get(self, '/sessions/%s/status' % (sessionId), restPort) == SessionStates.RUNNING:
time.sleep(0.2)
self.assertEquals(SessionStates.FINISHED, testutils.get(self, '/sessions/%s/status' % (sessionId), restPort))
testutils.delete(self, '/sessions/%s' % (sessionId), restPort)
# We put an NGAS archiving at the end of the chain, let's check that the DROPs were copied over there
# Since the graph consists on several SleepAndCopy apps, T should contain the message repeated
# 9 times, and S should have it 4 times
def checkReplica(dropId, copies):
response = ngaslite.retrieve('ngas.ddns.net', dropId)
buff = response.read()
self.assertEquals(msg*copies, buff, "%s's replica doesn't look correct" % (dropId))
checkReplica('T%s' % (suffix), 9)
checkReplica('S%s' % (suffix), 4)