def test_StatusCmd_1(self):
"""
Synopsis:
Issue STATUS Command/basic (no parameters).
Description:
Issue a STATUS Command via the C-Client (on the shell).
Expected Result:
The STATUS Command should be accepted and executed by the C-Client
and the server.
Test Steps:
- Start standard server.
- Execute the ngamsCClient tool on the shell.
- Capture the output from the ngamsCClient and compare this with the
expected output.
Remarks:
...
"""
cfgObj, dbObj = self.prepExtSrv(port=8000)
statObj = _execCClient(pars=[["-port", "8000"], ["-cmd", "STATUS"]])[0]
refStatFile = "ref/ngamsCClientTest_test_StatusCmd_1_1_ref"
refStatFile = saveInFile(
None,
loadFile(refStatFile) % (getHostName() + ":8000"))
tmpStatFile = saveInFile(
None, filterOutLines(statObj.dumpBuf(), ["Date", "Version"]))
self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect info in " +\
"STATUS Command XML Status Document")
def handleHttpRequest(self, *args, **kwargs):
if os.path.exists("tmp/handleHttpRequest_tmp"):
reqHandleCode = loadFile("tmp/handleHttpRequest_tmp")
reqHandleCode = getattr(self, reqHandleCode)
reqHandleCode(*args, **kwargs)
else:
super(ngamsServerTestDynReqCallBack, self).handleHttpRequest(*args, **kwargs)
def reqCallBack(self, *args, **kwargs):
"""
Override ngamsServer.reqCallBack(). Simply load the name of the
request handler to execute in the file written by the test case.
"""
if os.path.exists("tmp/reqCallBack_tmp"):
reqHandleCode = loadFile("tmp/reqCallBack_tmp")
reqHandleCode = getattr(self, reqHandleCode)
reqHandleCode(*args, **kwargs)
else:
super(ngamsServerTestDynReqCallBack, self).reqCallBack(*args, **kwargs)
def test_WakeUpStatus_1(self):
"""
Synopsis:
Test that suspended server is woken up when
STATUS?host_id=<Sub-Node> specifying that node is sent.
Description:
If a STATUS Command is send to a node requesting status from another
node (Proxy Mode), and the target is suspended, the contacted node
must wake up the suspended target node. The purpose of this test
is to verify this use case.
Expected Result:
The contacted node should identify the target node as suspended,
and should wake it up prior to forwarding the STATUS Command acting
as proxy.
Test Steps:
- Start simulated cluster specifying that one node can suspend itself.
- Wait till sub-node has suspended itself.
- Send STATUS Command to the simulated master requesting status from
the suspended sub-node.
- Check the response to the STATUS Command.
- Cross check that the suspended node is now woken up by sending
a STATUS Command directly to the sub-node.
Remarks:
...
"""
cfgParDic = {"8001": [["%s.IdleSuspensionTime" % SUSP_EL, "5"]]}
dbConObj = prepSimCluster(self, cfgParDic=cfgParDic)[masterNode][1]
self.waitTillSuspended(dbConObj, subNode1, 30, susp_nodes)
# 1. Send STATUS Command to sub-node using master as proxy.
statObj = sendPclCmd(port=8000, auth=AUTH).\
get_status(NGAMS_STATUS_CMD, pars = [["host_id", subNode1]])
statBuf = filterOutLines(statObj.dumpBuf(), ["Date:", "Version:"])
tmpStatFile = saveInFile(None, statBuf)
refStatFile = "ref/ngamsIdleSuspensionTest_test_WakeUpStatus_1_1_ref"
refStatFile = saveInFile(None, loadFile(refStatFile) % getHostName())
self.checkFilesEq(refStatFile, tmpStatFile,
"Sub-node not woken up as expected")
# 2. Double-check that sub-node is no longer suspended.
statObj = sendPclCmd(port=8001, auth=AUTH).status()
statBuf = filterOutLines(statObj.dumpBuf(), ["Date:", "Version:"])
tmpStatFile = saveInFile(None, statBuf)
self.checkFilesEq(refStatFile, tmpStatFile,
"Sub-node not woken up as expected")
# Clean up.
self.markNodesAsUnsusp(dbConObj, susp_nodes)
def test_WakeUpStatus_2(self):
"""
Synopsis:
Test that suspended server is woken up when STATUS?file_access is
issued, specifying a file on a suspended sub-node.
Description:
The purpose of the test is to verify that a suspended sub-node is
woken up by a master node acting as proxy if a STATUS?file_id is
send pointing to a file on the suspended sub-node.
Expected Result:
The suspended sub-node should be woken up by the master node which
after the suspended sub-node has come Online should forward the
request which is executed on the sub-node and the result send back to
the client through the master node.
Test Steps:
- Prepare simulated cluster with unit, which suspends itself.
- Archive 3 files onto sub-node.
- Wait till sub-node has suspended itself.
- Send a STATUS?file_id request to the Master Node specifying one
of the files previously archived.
- Check the result returned from the sub-node via the Master Node.
Remarks:
...
"""
cfgParDic = {"8001": [["%s.IdleSuspensionTime" % SUSP_EL, "5"]]}
dbConObj = prepSimCluster(self, cfgParDic=cfgParDic)[masterNode][1]
# Archive some files on the two nodes and wait till sub-node
# has suspended itself.
sendPclCmd(port=8000, auth=AUTH).archive("src/TinyTestFile.fits")
sendPclCmd(port=8000, auth=AUTH).archive("src/SmallFile.fits")
sendPclCmd(port=8001, auth=AUTH).archive("src/SmallFile.fits")
self.waitTillSuspended(dbConObj, subNode1, 20, susp_nodes)
# Retrieve information about the file on the suspended sub-node.
fileId = "TEST.2001-05-08T15:25:00.123"
statObj = sendPclCmd(port=8000, auth=AUTH).\
get_status(NGAMS_STATUS_CMD, pars=[["file_access", fileId]])
statBuf = filterOutLines(statObj.dumpBuf(), ["Date:", "Version:"])
tmpStatFile = saveInFile(None, statBuf)
refStatFile = "ref/ngamsIdleSuspensionTest_test_WakeUpStatus_2_1_ref"
refStatFile = saveInFile(None, loadFile(refStatFile) %\
(getHostName(), getHostName()))
self.checkFilesEq(refStatFile, tmpStatFile,
"Unexpected reply to STATUS?file_access request")
def reqCallBack_IllegalResp(self, *args, **kwargs):
"""
Override ngamsServer.reqCallBack(). This version simply writes an
illegal HTTP response.
Only commands that are handled are: EXIT, OFFLINE, STATUS.
"""
path = kwargs['path'] if 'path' in kwargs else args[3]
req_handler = kwargs['httpRef'] if 'httpRef' in kwargs else args[0]
cmd = path.strip().split('?')[0]
if cmd in (NGAMS_EXIT_CMD, NGAMS_OFFLINE_CMD, NGAMS_STATUS_CMD):
super(ngamsServerTestDynReqCallBack, self).reqCallBack(*args, **kwargs)
else:
# Sending back illegal HTTP response
time.sleep(0.500)
resp = loadFile("tmp/ngamsServerTestIllegalResp_tmp", mode='b')
req_handler.wfile.write(resp)
def test_NormalExec_1(self):
"""
Synopsis:
Normal execution of CHECKFILE Command.
Description:
The purpose of this Text Case is to test the normal/standard
execution of the CHECKFILE Command.
Expected Result:
The CHECKFILE Command should be accepted and executed successfully
by the server on an existing file.
Test Steps:
- Start NG/AMS Server.
- Archive a file successfully.
- Check the file specifying Disk ID/File Id/File Version.
- Check the the result returned by the NG/AMS Server is as expected.
Remarks:
...
"""
self.prepExtSrv()
sendPclCmd().archive("src/SmallFile.fits")
diskId = "tmp-ngamsTest-NGAS-FitsStorage1-Main-1"
fileId = "TEST.2001-05-08T15:25:00.123"
statObj = sendPclCmd(port=8888).get_status(NGAMS_CHECKFILE_CMD,
pars=[["disk_id", diskId],
["file_id", fileId],
["file_version",
"1"]])
refStatFile = "ref/ngamsCheckFileCmdTest_test_NormalExec_1_1_ref"
refStatFile = saveInFile(
None,
loadFile(refStatFile) % ("%s:%d" % (getHostName(), 8888)))
tmpStatFile = saveInFile(None, statObj.getMessage())
self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect info in " +\
"STATUS Command XML Status Document")
def test_WakeUpDataCheck_1(self):
"""
Synopsis:
Check that a suspended sub-node is woken up when DCC is due and that
the DCC is executed as expected after the node has been woken up.
Description:
Before suspending itself, a sub-node should request to be woken up
when the time for executing the next DCC is due.
The purpose of the test is to verify that a sub-node suspending itself
requests to be woken up at the specified point in time, and that the
contacted Master Node wakes up a suspended sub-node as expected.
Expected Result:
The contacted Master Node should identify that the sub-node is
suspended and that it has requested to be woken up by the Master Node
at a given point in time. The Master Node should wake up the node at
the requested point in time and the sub-node should carry out the DCC.
Test Steps:
- Prepare simulated cluster with a sub-node suspending itself after a
short while and having DCC enabled with a high frequency.
- Archive two files onto the sub-node.
- Wait till the sub-node has suspended itself.
- Wait till the sub-node has been woken up.
- Check that entries in the log file on the Master Node indicate
that it has woken up the suspended sub-node.
- Check that entries in the log file on the sub-node indicate that it
has been woken up.
- Wait until the DCC has been executed and check that the result is
as expected (the DCC summary log appears in the log file on the
sub-node).
Remarks:
...
"""
# Enable DCC + define a minimum check cycle of 15s.
cfgParDic = {
"8001": [["%s.IdleSuspensionTime" % SUSP_EL, "5"],
["NgamsCfg.DataCheckThread[1].Active", "1"],
["NgamsCfg.DataCheckThread[1].MinCycle", "00T00:00:15"]]
}
# Always delete sub-node log file (to run test with a fresh log file).
rmFile(subNode1Log)
dbConObj = prepSimCluster(self, cfgParDic=cfgParDic)[masterNode][1]
sendPclCmd(port=8001, auth=AUTH).archive("src/TinyTestFile.fits")
sendPclCmd(port=8001, auth=AUTH).archive("src/SmallFile.fits")
self.waitTillSuspended(dbConObj, subNode1, 30, susp_nodes)
# Get timestamp for the log indicating that the node suspends itself.
suspLog = "NG/AMS Server %s suspending itself" % subNode1
subNodeLog = loadFile(subNode1Log).split("\n")
suspLogEntry = ""
for line in subNodeLog:
if (line.find(suspLog) != -1):
suspLogEntry = line
break
if (not suspLogEntry):
self.fail("Did not find expected log entry in sub-node " +\
"log file: " + suspLog)
suspLogTime = suspLogEntry.split(" ")[0]
# Now we have to wait until the sub-node is woken up for DCC.
self.waitTillWokenUp(dbConObj, subNode1, 60, susp_nodes)
# Check log output in Master Log File.
tagFormat = "Waking up server %s"
testTags = [tagFormat % (subNode1, )]
self.checkTags(loadFile(masterNodeLog), testTags, showBuf=0)
# Check log output in Sub-Node Log File.
sendPclCmd(port=8001, auth=AUTH).status() # Flush log cache.
time.sleep(2) # and wait a bit
testTags = ["NGAS Node: %s woken up after" % subNode1]
self.checkTags(loadFile(subNode1Log), testTags, showBuf=0)
# Wait until the DCC has been executed.
startTime = time.time()
dccStatLog = ""
while ((time.time() - startTime) < 60):
subNodeLogBuf = loadFile(subNode1Log)
subNodeLogBufList = subNodeLogBuf.split("\n")
subNodeLogBufList.reverse()
for line in subNodeLogBufList:
if (line.find("NGAMS_INFO_DATA_CHK_STAT") != -1):
dccStatLog = line
break
time.sleep(0.100)
dccStatLogTime = dccStatLog.split(" ")[0]
if ((not dccStatLog) or (dccStatLogTime < suspLogTime)):
self.fail("Data Consistency Checking not executed AFTER host " +\
"wake-up as expected")
testTags = ["NGAMS_INFO_DATA_CHK_STAT:3020:INFO: Number of files " +\
"checked: 4"]
self.checkTags(subNodeLogBuf, testTags, showBuf=0)
def test_WakeUpCheckfile_1(self):
"""
Synopsis:
Check that CHECKFILE?file_id&file_version correctly handled
when the referenced file is stored on a suspended node.
Description:
The purpose of the test is to verify that a CHECKFILE Command is
properly handled by a Master node acting as a proxy and interacting
with a suspended sub-node.
Expected Result:
The contacted Master Node should identify that the sub-node hosting
the specified file is suspended, and should wake it up before
forwarding the request. Subsequently it should forward the request to
the sub-node and send back the result to the client.
Test Steps:
- Start simulated cluster with a sub-node suspending itself after a
short while.
- Archive 2 FITS file 2 times each.
- Wait till the sub-node suspends itself.
- Send a CHECKFILE Command to the Master node specifying to check
one of the previously archived files.
- Check response from the sub-node.
- Check from the log entries in the Master Node that the CHECKFILE
Command has been successfully executed.
- Check from the log entries on the sub-node that the CHECKFILE
Command has been successfully executed.
Remarks:
...
"""
cfgParDic = {"8001": [["%s.IdleSuspensionTime" % SUSP_EL, "5"]]}
dbConObj = prepSimCluster(self, cfgParDic=cfgParDic)[masterNode][1]
sendPclCmd(port=8001, auth=AUTH).archive("src/TinyTestFile.fits")
sendPclCmd(port=8001, auth=AUTH).archive("src/SmallFile.fits")
sendPclCmd(port=8000, auth=AUTH).archive("src/TinyTestFile.fits")
sendPclCmd(port=8000, auth=AUTH).archive("src/SmallFile.fits")
self.waitTillSuspended(dbConObj, subNode1, 10, susp_nodes)
# Execute CHECKFILE Command on a file on the suspended sub-node.
file_id = "TEST.2001-05-08T15:25:00.123"
cmdPars = [["file_id", file_id], ["file_version", "1"]]
statObj = sendPclCmd(port=8000, auth=AUTH).\
get_status(NGAMS_CHECKFILE_CMD, pars=cmdPars)
# Check that request response is as expected.
statBuf = filterOutLines(statObj.dumpBuf(), ["Date:", "Version:"])
tmpStatFile = saveInFile(None, statBuf)
refStatFile = "ref/ngamsIdleSuspensionTest_test_WakeUpCheckfile_1_1_ref"
refStatFile = saveInFile(None, loadFile(refStatFile) %\
(subNode1, subNode1))
self.checkFilesEq(refStatFile, tmpStatFile,"CHECKFILE Command not " +\
"executed on sub-node as expected")
#tstStr = "NGAMS_INFO_FILE_OK:4056:INFO: Checked file with File ID: "+\
# "TEST.2001-05-08T15:25:00.123/File Version: 1/Disk ID"
#if (statObj.getMessage().find(tstStr) == -1):
#self.fail("CHECKFILE Command not executed on sub-node as expected")
# Check that expected log entries found in the Master Node Log File.
refStatFile = "ref/ngamsIdleSuspensionTest_test_WakeUpCheckfile_1_2_ref"
testTags = loadFile(refStatFile) % (subNode1, )
sendPclCmd(port=8000, auth=AUTH).status()
masterLogBuf = loadFile(masterNodeLog)
self.checkTags(masterLogBuf, testTags, showBuf=0)
# Check that expected log entries found in the Sub-Node Log File.
tmpTag = "File list to check: (1: Location:LOCAL, Host:%s, " +\
"Version:1) (2: Location:LOCAL, Host:%s, Version:1)"
testTags = [('CHECKFILE?file_id=' + urllib.quote(file_id) +
'&file_version=1&time_out=60.0'),
tmpTag % (subNode1, subNode1)]
subNodeLogBuf = loadFile(subNode1Log)
self.checkTags(subNodeLogBuf, testTags, showBuf=0)