def test_WakeUpRetrieve_1(self):
"""
Synopsis:
Check that RETRIEVE?file_id and RETRIEVE?file_id&file_version are
correctly handled when the file specified is stored on suspended node.
Description:
The test verifies that a suspended sub-node is woken up when a
RETRIEVE Request is send to the Master Node and the file properly
retrieved and send back to the client via the Master Node acting
as proxy.
Expected Result:
The Master Node server should identify the target node as suspended
and should wake it up before requesting the data. Subsequently the
data should be send back to the client by the Master Node acting
as proxy.
Test Steps:
- Start simulated cluster with a sub-node suspending itself after
a short while.
- Archive a small FITS file 3 times onto the sub-node.
- Wait till the sub-node has suspended itself.
- Submit a REQUEST?file_id to the Master Node.
- Check that the response is as expected.
- Check that the file has arrived on disk.
Remarks:
TODO!: Check that the file has arrived on disk as expected.
"""
cfgParDic = {"8001": [["%s.IdleSuspensionTime" % SUSP_EL, "5"]]}
dbConObj = prepSimCluster(self, cfgParDic=cfgParDic)[masterNode][1]
for n in range(3):
sendPclCmd(port=8001, auth=AUTH).archive("src/SmallFile.fits")
# Retrieve the file as file_id, file_id/file_version.
testParsList = [["TEST.2001-05-08T15:25:00.123", -1],
["TEST.2001-05-08T15:25:00.123", 2]]
for fileId, version in testParsList:
self.waitTillSuspended(dbConObj, subNode1, 20, susp_nodes)
# different prefixes to avoid name clashes
tmpRetFile = genTmpFilename("original_")
unzippedRetFile = genTmpFilename("unzip_")
statObj = sendPclCmd(port=8000, auth=AUTH).\
retrieve(fileId, fileVersion=version,
targetFile=tmpRetFile)
self.checkEqual(NGAMS_SUCCESS, statObj.getStatus(),
"Unexpected return value for RETRIEVE Command")
unzip(tmpRetFile, unzippedRetFile)
self.checkFilesEq("src/SmallFile.fits", unzippedRetFile,
"File retrieved incorrect")
def test_RetrieveCmd_1(self):
"""
Synopsis:
Retrieve file from server hosting the file.
Description:
The purpose of the test is to test the case where it is attempted
to retrieve an archived file directly from the unit hosting the file.
Expected Result:
The server should locate the file and send it back to the requestor.
Test Steps:
- Start server.
- Archive FITS file.
- Retrieve the file and store it in a local file.
- Check reply from the server indiates that the request was handled
successfully.
- Check that the file on disk has been successfully retrieved.
Remarks:
...
"""
self.prepExtSrv()
client = sendPclCmd()
client.archive("src/SmallFile.fits")
# Retrieve the file.
trgFile = "tmp/test_RetrieveCmd_1_1_tmp"
outFilePath = "tmp/SmallFile.fits"
status = client.retrieve("TEST.2001-05-08T15:25:00.123",
targetFile=trgFile)
unzip(trgFile, outFilePath)
# Check reply.
refStatFile = "ref/ngamsRetrieveCmdTest_test_RetrieveCmd_1_1_ref"
tmpStatFile = "tmp/ngamsRetrieveCmdTest_test_RetrieveCmd_1_1_tmp"
saveInFile(tmpStatFile, filterDbStatus1(status.dumpBuf(0, 1, 1)))
self.checkFilesEq(refStatFile, tmpStatFile,
"Incorrect status for RETRIEVE Command/Normal " +\
"Execution")
# Check file retrieved.
self.checkFilesEq("src/SmallFile.fits", outFilePath,
"Retrieved file incorrect")
def test_Retrieve2File_1(self):
"""
Synopsis:
Handling of RETRIEVE Command via P-Client.
Description:
Test correct handling of the RETRIEVE command from the NG/AMS
Python API (no processing).
Expected Result:
The RETRIEVE Command should be send to the server requesting for
an archive file, which subsequently should be send back to the client
and stored on the local disk.
Test Steps:
- Start server.
- Archive a file.
- Retrieve the archived file into a local file.
- Check the response from the server.
- Check that the requested file has been successfully stored on the
local disk.
Remarks:
...
"""
self.prepExtSrv()
client = ngamsPClient.ngamsPClient(port=8888)
client.archive("src/SmallFile.fits")
trgDir = "tmp"
status = client.retrieve("TEST.2001-05-08T15:25:00.123",
targetFile=trgDir)
refMsg = "Successfully handled request"
self.checkEqual(refMsg, status.getMessage(), "Problem executing " +\
"RETRIEVE Command")
tmpFile = "tmp/TEST.2001-05-08T15:25:00.123.fits.gz"
unzipedTmpFile = 'tmp/SmallFile.fits'
unzip(tmpFile, unzipedTmpFile)
self.checkFilesEq("src/SmallFile.fits", unzipedTmpFile,
"Retrieved file incorrect")
def test_VolumeDir_01(self):
"""
Synopsis:
Grouping of data volumes under the Volume Dir in the NGAS Root Dir.
Description:
See ngamsArchiveCmdTest.test_VolumeDir_01().
This tests verifies that the files archived into this structure can
be received.
Expected Result:
When the server goes Online, it should accept the given directory
structure and it should be possible to archive files into this
structureand subsequently to receive them.
Test Steps:
- Create the volume dirs from an existing structure.
- Start server with configuration specifying the Volumes Dir in which
all volumes will be hosted.
- Archive a FITS file.
- Retrieve the files and check that they are OK.
Remarks:
...
"""
# Create basic structure.
ngasRootDir = "/tmp/ngamsTest/NGAS/"
rmFile(ngasRootDir)
checkCreatePath(ngasRootDir)
subprocess.check_call(['tar', 'zxf', 'src/volumes_dir.tar.gz'])
mvFile('volumes', ngasRootDir)
# Create configuration, start server.
cwd = os.getcwd()
configFile = os.path.normpath(cwd +
"/src/ngamsCfg_VolumeDirectory.xml")
self.prepExtSrv(delDirs=0, cfgFile=configFile)
client = sendPclCmd()
# Archive a file.
stat = client.archive("src/SmallFile.fits")
# dpallot: this will always fail on the mac as the tar sizes are different
# to the hard coded test results in the old file:
# ngamsRetrieveCmdTest_test_VolumeDir_01_01_ref
#tmpStatFile = saveInFile(None, filterDbStatus1(stat.dumpBuf()))
#refStatFile = "ref/ngamsRetrieveCmdTest_test_VolumeDir_01_01_ref"
#self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect status " +\
# "message from NG/AMS Server")
self.assertEquals(stat.getStatus(), 'SUCCESS')
# Check that the target files have been archived in their
# appropriate locations.
trgFile = "tmp/test_VolumeDir_01_tmp"
refFile = "src/SmallFile.fits"
outFilePath = "tmp/SmallFile.fits"
stat = client.retrieve("TEST.2001-05-08T15:25:00.123",
targetFile=trgFile)
self.assertEqual(NGAMS_SUCCESS, stat.getStatus(), stat.getMessage())
# unzip the the file and diff against original
unzip(trgFile, outFilePath)
self.checkFilesEq(outFilePath, refFile, "Retrieved file incorrect")
def test_CloneCmd_1(self):
"""
Synopsis:
Normal execution of CLONE Command/clone one file/async=1.
Description:
Test normal execution of the CLONE Command whereby async=1.
Expected Result:
An immediate response should be returned indicating that the
CLONE Command has been accepted for execution. The Clone Status
Report should be send out indicating that the file was cloned.
Test Steps:
- Start 1 NG/AMS Server.
- Archive file 2 times.
- Clone one file specifying disk_id, file_id and file_version + async=1.
- Check that the immediate response is correctly returned.
- Wait for the execution of the CLONE Command to finish.
- Check that the Request Info in the NG/AMS Server indicates that the
Clone Request finished.
- Check that the cloned file has arrived on the Target Disk.
- Check that the DB info has been updated as it should.
Remarks:
TODO: Re-implement using _execCloneTest().
"""
srcFile = "src/SmallFile.fits"
cfgObj, dbObj = self.prepExtSrv(cfgProps=(('NgamsCfg.Server[1].RequestDbBackend', 'memory'),))
client = sendPclCmd()
for n in range(2): client.archive(srcFile)
flushEmailQueue()
testUserEmail = getpass.getuser()+"@"+ngamsLib.getCompleteHostName()
statObj = client.get_status(NGAMS_CLONE_CMD,
pars = [["disk_id", srcDiskId],
["file_id", nmuFileId],
["file_version", "1"],
["async", "1"],
["notif_email", testUserEmail]])
refStatFile = "ref/ngamsCloneCmdTest_test_CloneCmd_1_1_ref"
tmpStatFile = saveInFile(None, filterDbStatus1(statObj.dumpBuf()))
self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect status for " +\
"CLONE command/successfull cloning")
finalStatObj = waitReqCompl(client, statObj.getRequestId())
complPer = str(finalStatObj.getCompletionPercent())
self.checkEqual("100.0", complPer,
genErrMsgVals("Incorrect Request Status for CLONE " +\
"Command/Completion Percent", "100.0",
complPer))
if _checkMail:
mailCont = getEmailMsg(["NGAS Host:", "Total proc", "Handling time"])
tmpStatFile = "tmp/ngamsCloneCmdTest_test_CloneCmd_1_tmp"
refStatFile = "ref/ngamsCloneCmdTest_test_CloneCmd_1_ref"
saveInFile(tmpStatFile, mailCont)
self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect/missing " +\
"CLONE Status Notification Email Msg")
tmpFitsFile = "/tmp/ngamsTest/NGAS/FitsStorage2-Main-3/saf/" +\
"2001-05-08/1/TEST.2001-05-08T15:25:00.123.fits.gz"
unzippedTmp = genTmpFilename()
unzip(tmpFitsFile, unzippedTmp)
self.checkFilesEq(srcFile, unzippedTmp, "Incorrect cloned file generated")
diskId = "tmp-ngamsTest-NGAS-FitsStorage2-Main-3"
filePrefix = "ngamsCloneCmdTest_test_CloneCmd_1"
fileInfoRef = "ref/" + filePrefix + "_FileInfo_ref"
fileInfoTmp = "tmp/" + filePrefix + "_FileInfo_tmp"
fileInfo = ngamsFileInfo.\
ngamsFileInfo().read(getHostName() + ":8888",
dbObj, "TEST.2001-05-08T15:25:00.123",
1, diskId)
saveInFile(fileInfoTmp, filterDbStatus1(fileInfo.dumpBuf()))
self.checkFilesEq(fileInfoRef, fileInfoTmp, "Incorrect info in DB " +\
"for cloned file")