def test_WakeUpRetrieve_1(self):
"""
Synopsis:
Check that RETRIEVE?file_id and RETRIEVE?file_id&file_version are
correctly handled when the file specified is stored on suspended node.
Description:
The test verifies that a suspended sub-node is woken up when a
RETRIEVE Request is send to the Master Node and the file properly
retrieved and send back to the client via the Master Node acting
as proxy.
Expected Result:
The Master Node server should identify the target node as suspended
and should wake it up before requesting the data. Subsequently the
data should be send back to the client by the Master Node acting
as proxy.
Test Steps:
- Start simulated cluster with a sub-node suspending itself after
a short while.
- Archive a small FITS file 3 times onto the sub-node.
- Wait till the sub-node has suspended itself.
- Submit a REQUEST?file_id to the Master Node.
- Check that the response is as expected.
- Check that the file has arrived on disk.
Remarks:
TODO!: Check that the file has arrived on disk as expected.
"""
cfgParDic = {"8001": [["%s.IdleSuspensionTime" % SUSP_EL, "5"]]}
dbConObj = prepSimCluster(self, cfgParDic=cfgParDic)[masterNode][1]
for n in range(3):
sendPclCmd(port=8001, auth=AUTH).archive("src/SmallFile.fits")
# Retrieve the file as file_id, file_id/file_version.
testParsList = [["TEST.2001-05-08T15:25:00.123", -1],
["TEST.2001-05-08T15:25:00.123", 2]]
for fileId, version in testParsList:
self.waitTillSuspended(dbConObj, subNode1, 20, susp_nodes)
# different prefixes to avoid name clashes
tmpRetFile = genTmpFilename("original_")
unzippedRetFile = genTmpFilename("unzip_")
statObj = sendPclCmd(port=8000, auth=AUTH).\
retrieve(fileId, fileVersion=version,
targetFile=tmpRetFile)
self.checkEqual(NGAMS_SUCCESS, statObj.getStatus(),
"Unexpected return value for RETRIEVE Command")
unzip(tmpRetFile, unzippedRetFile)
self.checkFilesEq("src/SmallFile.fits", unzippedRetFile,
"File retrieved incorrect")
def test_DppiProc_02(self):
"""
Synopsis:
Test the proper execution of DPPI processing/result in buffer.
Description:
When requesting a file from NGAS, it is possible to specify to have
the file processed by a DPPI. The result can either be stored in a
file or contained in the buffer when handed over to the NG/AMS Server.
Expected Result:
The DPPI should be invoked the file to be retrieved and the result
(stored by the DPPI in a buffer), properly sent back to the client.
Test Steps:
- Start a server configured to invoke a test DPPI (header extraction).
- Archive a file.
- Retrieve the file, specifying to apply the DPPI on it. Result will
be stored in a buffer in memory.
- Check that the file has been processed as expected.
Remarks:
...
Test Data:
...
"""
tmpCfgFile = genTmpFilename("ngamsRetrieveCmdTest")
cfg = ngamsConfig.ngamsConfig().load("src/ngamsCfg.xml")
cfg.storeVal("NgamsCfg.Processing[1].PlugIn[1].Name",
"ngamsTest.ngamsTestDppi1")
cfg.storeVal("NgamsCfg.Processing[1].PlugIn[1].PlugInPars",
"TAG=test_DppiProc_02,TARGET=BUFFER")
cfg.save(tmpCfgFile, 0)
self.prepExtSrv(cfgFile=tmpCfgFile)
client = sendPclCmd()
client.archive("src/SmallFile.fits")
# Retrieve the file specifying to apply the DPPI.
outFile = genTmpFilename("test_DppiProc_02")
pars = [["test_suite", "ngamsRetrieveCmdTest"],
["test_case", "test_DppiProc_02"]]
stat = client.retrieve("TEST.2001-05-08T15:25:00.123",
targetFile=outFile,
processing="ngamsTest.ngamsTestDppi1",
pars=pars)
refStatFile = "ref/ngamsRemFileCmdTest_test_DppiProc_02_01_ref"
self.checkFilesEq(refStatFile, outFile, "Incorrect status for " +\
"RETRIEVE Command/DPPI Processing, result in buffer")
def test_DppiProc_03(self):
"""
Synopsis:
Test the proper execution of DPPI proc./result in file/Proxy Mode.
Description:
When requesting a file from NGAS, it is possible to specify to have
the file processed by a DPPI. The result can either be stored in a
file or contained in the buffer when handed over to the NG/AMS Server.
Expected Result:
The DPPI should be invoked the file to be retrieved and the result
(stored by the DPPI in a file), properly sent back to the client.
This is tested while retrieving the DPPI processed via a server acting
as proxy.
Test Steps:
- Start a simulated cluster.
- Archive a file into sub-node.
- Retrieve the file via the master, specifying to apply the DPPI on it.
- Check that the file has been processed as expected.
Remarks:
...
Test Data:
...
"""
ncuCfgPars = [[
"NgamsCfg.Processing[1].PlugIn[1].Name", "ngamsTest.ngamsTestDppi1"
],
[
"NgamsCfg.Processing[1].PlugIn[1].PlugInPars",
"TAG=test_DppiProc_02,TARGET=FILE"
]]
self.prepCluster((8000, (8011, ncuCfgPars)))
stat = sendPclCmd(port=8011).archive("src/SmallFile.fits")
self.assertEquals('SUCCESS', stat.getStatus())
# Retrieve the file specifying to apply the DPPI.
outFile = genTmpFilename("test_DppiProc_03")
pars = [["test_suite", "ngamsRetrieveCmdTest"],
["test_case", "test_DppiProc_03"]]
stat = sendPclCmd(port=8000).retrieve(
"TEST.2001-05-08T15:25:00.123",
targetFile=outFile,
processing="ngamsTest.ngamsTestDppi1",
pars=pars)
refStatFile = "ref/ngamsRemFileCmdTest_test_DppiProc_03_01_ref"
self.checkFilesEq(refStatFile, outFile, "Incorrect status for " +\
"RETRIEVE Command/DPPI Processing, Proxy Mode, " +\
"result in buffer")
def test_CloneCmd_1(self):
"""
Synopsis:
Normal execution of CLONE Command/clone one file/async=1.
Description:
Test normal execution of the CLONE Command whereby async=1.
Expected Result:
An immediate response should be returned indicating that the
CLONE Command has been accepted for execution. The Clone Status
Report should be send out indicating that the file was cloned.
Test Steps:
- Start 1 NG/AMS Server.
- Archive file 2 times.
- Clone one file specifying disk_id, file_id and file_version + async=1.
- Check that the immediate response is correctly returned.
- Wait for the execution of the CLONE Command to finish.
- Check that the Request Info in the NG/AMS Server indicates that the
Clone Request finished.
- Check that the cloned file has arrived on the Target Disk.
- Check that the DB info has been updated as it should.
Remarks:
TODO: Re-implement using _execCloneTest().
"""
srcFile = "src/SmallFile.fits"
cfgObj, dbObj = self.prepExtSrv(cfgProps=(('NgamsCfg.Server[1].RequestDbBackend', 'memory'),))
client = sendPclCmd()
for n in range(2): client.archive(srcFile)
flushEmailQueue()
testUserEmail = getpass.getuser()+"@"+ngamsLib.getCompleteHostName()
statObj = client.get_status(NGAMS_CLONE_CMD,
pars = [["disk_id", srcDiskId],
["file_id", nmuFileId],
["file_version", "1"],
["async", "1"],
["notif_email", testUserEmail]])
refStatFile = "ref/ngamsCloneCmdTest_test_CloneCmd_1_1_ref"
tmpStatFile = saveInFile(None, filterDbStatus1(statObj.dumpBuf()))
self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect status for " +\
"CLONE command/successfull cloning")
finalStatObj = waitReqCompl(client, statObj.getRequestId())
complPer = str(finalStatObj.getCompletionPercent())
self.checkEqual("100.0", complPer,
genErrMsgVals("Incorrect Request Status for CLONE " +\
"Command/Completion Percent", "100.0",
complPer))
if _checkMail:
mailCont = getEmailMsg(["NGAS Host:", "Total proc", "Handling time"])
tmpStatFile = "tmp/ngamsCloneCmdTest_test_CloneCmd_1_tmp"
refStatFile = "ref/ngamsCloneCmdTest_test_CloneCmd_1_ref"
saveInFile(tmpStatFile, mailCont)
self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect/missing " +\
"CLONE Status Notification Email Msg")
tmpFitsFile = "/tmp/ngamsTest/NGAS/FitsStorage2-Main-3/saf/" +\
"2001-05-08/1/TEST.2001-05-08T15:25:00.123.fits.gz"
unzippedTmp = genTmpFilename()
unzip(tmpFitsFile, unzippedTmp)
self.checkFilesEq(srcFile, unzippedTmp, "Incorrect cloned file generated")
diskId = "tmp-ngamsTest-NGAS-FitsStorage2-Main-3"
filePrefix = "ngamsCloneCmdTest_test_CloneCmd_1"
fileInfoRef = "ref/" + filePrefix + "_FileInfo_ref"
fileInfoTmp = "tmp/" + filePrefix + "_FileInfo_tmp"
fileInfo = ngamsFileInfo.\
ngamsFileInfo().read(getHostName() + ":8888",
dbObj, "TEST.2001-05-08T15:25:00.123",
1, diskId)
saveInFile(fileInfoTmp, filterDbStatus1(fileInfo.dumpBuf()))
self.checkFilesEq(fileInfoRef, fileInfoTmp, "Incorrect info in DB " +\
"for cloned file")