def test_RemDiskCmd_1(self):
"""
Synopsis:
Normal execution.
Description:
Test the normal execution of the REMDISK Command, where a disk,
containing files with at least 3 independent copies is request
REMDISK'ed.
Expected Result:
When executing the REMDISK Command on the cloned disk with execute=0,
the server should accept the command and report this to the client.
When re-issuing the REMDISK Command on the cloned disk with execute=1,
the server should accept the command, execute the REMDISK procedure on
the disk and report this to the client.
Test Steps:
- Start server.
- Archive a file.
- Clone the disk hosting the archived file.
- Issue REMDISK Command to remove the cloned disk (execute=0).
- Check response from the server.
- Issue REMDISK Command to remove the cloned disk (execute=1).
- Check response from the server.
Remarks:
TODO!: It is not checked that the contents on the disk and the info
in the DB in ngas_files and ngas_disks is properly updated.
"""
self.prepExtSrv(cfgProps=(('NgamsCfg.Server[1].RequestDbBackend', 'memory'),))
client = sendPclCmd()
# Archive a file + clone it to be able to execute the REMDISK Command.
diskId = "tmp-ngamsTest-NGAS-FitsStorage1-Main-1"
client.archive("src/SmallFile.fits")
status = client.clone("", diskId, -1)
waitReqCompl(client, status.getRequestId())
# Remove the cloned disk (execute=0), should be successfull.
status = client.remDisk(diskId, 0)
refStatFile = "ref/ngamsRemDiskCmdTest_test_RemDiskDisk_1_1_ref"
tmpStatFile = "tmp/ngamsRemDiskCmdTest_test_RemDiskDisk_1_1_tmp"
saveInFile(tmpStatFile, filterDbStatus1(status.dumpBuf(0, 1, 1)))
self.checkFilesEq(refStatFile, tmpStatFile,
"Incorrect status for REMDISK Command/no execution")
# Remove the cloned disk (execute=1), should be successfull.
status = client.remDisk(diskId, 1)
refStatFile = "ref/ngamsRemDiskCmdTest_test_RemDiskDisk_1_2_ref"
tmpStatFile = "tmp/ngamsRemDiskCmdTest_test_RemDiskDisk_1_2_tmp"
saveInFile(tmpStatFile, filterDbStatus1(status.dumpBuf(0, 1, 1)))
self.checkFilesEq(refStatFile, tmpStatFile,
"Incorrect status for REMDISK Command/execution")
def test_RemFileCmd_2(self):
"""
Synopsis:
Missing file copies, REMFILE Command rejected.
Description:
The purpose of the Test Case is to verify that the REMFILE Command
is rejected when it is attempted to remove files available in less
than 3 copies in the archive. This both when specifying execute=0 and
execute=1.
Expected Result:
The server should detect that the file requested to be REMFILE'd is
not available on the system in at least 3 independent copies.
Test Steps:
- Start server.
- Archive file.
- Submit REMFILE Command to remove copy of file on Main Disk
(execute=0).
- Check on response from the server that the request was rejected.
- Submit REMFILE Command to remove copy of file on Main Disk
(execute=1).
- Check on response from the server that the request was rejected.
Remarks:
TODO!: It is not check if an Email Notification is sent out listing
files, which could not be removed because they were available
in less than 3 copies.
"""
self.prepExtSrv()
client = sendPclCmd()
# Archive file.
client.archive("src/SmallFile.fits")
# Remove the archived file (execute=0), should fail.
fileId = "TEST.2001-05-08T15:25:00.123"
diskId = "tmp-ngamsTest-NGAS-FitsStorage1-Main-1"
status = client.remFile(diskId, fileId, 1, 0)
refStatFile = "ref/ngamsRemFileCmdTest_test_RemFileCmd_2_1_ref"
tmpStatFile = "tmp/ngamsRemFileCmdTest_test_RemFileCmd_2_1_tmp"
saveInFile(tmpStatFile, filterDbStatus1(status.dumpBuf(0, 1, 1)))
self.checkFilesEq(refStatFile, tmpStatFile,
"Incorrect status for REMFILE Command/no execution")
# Remove the cloned file (execute=1), should fail.
status = client.remFile(diskId, fileId, 1, 1)
refStatFile = "ref/ngamsRemFileCmdTest_test_RemFileCmd_2_2_ref"
tmpStatFile = "tmp/ngamsRemFileCmdTest_test_RemFileCmd_2_2_tmp"
saveInFile(tmpStatFile, filterDbStatus1(status.dumpBuf(0, 1, 1)))
self.checkFilesEq(refStatFile, tmpStatFile,
"Incorrect status for REMFILE Command/execution")
def test_RemFileCmd_1(self):
"""
Synopsis:
Test normal execution of the REMFILE Command.
Description:
Test the normal execution of the REMFILE Command, whereby a file is
requested to the REMFILE'd which is available in at least 3 copies.
Expected Result:
The server should identify that the file requested to be removed is
available in at least 3 copies and should accept the request.
Test Steps:
- Start server.
- Archive file.
- Clone Main Disk hosting archived file.
- Submit REMFILE Command requesting to remove cloned file (execute=0).
- Check on response from server that the request has been accepted.
- Submit REMFILE Command requesting to remove cloned file (execute=1).
- Check on response from server that the request has been accepted.
Remarks:
TODO!: It is not checked that the info for the file is actually
removed from the DB and from the disk.
"""
self.prepExtSrv(cfgProps=(('NgamsCfg.Server[1].RequestDbBackend',
'memory'), ))
client = sendPclCmd()
# Archive a file + clone it to be able to execute the REMFILE Command.
client.archive("src/SmallFile.fits")
fileId = "TEST.2001-05-08T15:25:00.123"
diskId = "tmp-ngamsTest-NGAS-FitsStorage1-Main-1"
status = client.clone(fileId, diskId, 1)
waitReqCompl(client, status.getRequestId())
# Remove the cloned file (execute=0), should be successfull.
status = client.remFile(diskId, fileId, 1, 0)
refStatFile = "ref/ngamsRemFileCmdTest_test_RemFileCmd_1_1_ref"
tmpStatFile = "tmp/ngamsRemFileCmdTest_test_RemFileCmd_1_1_tmp"
saveInFile(tmpStatFile, filterDbStatus1(status.dumpBuf(0, 1, 1)))
self.checkFilesEq(refStatFile, tmpStatFile,
"Incorrect status for REMFILE Command/no execution")
# Remove the cloned file (execute=1), should be successfull.
status = client.remFile(diskId, fileId, 1, 1)
refStatFile = "ref/ngamsRemFileCmdTest_test_RemFileCmd_1_2_ref"
tmpStatFile = "tmp/ngamsRemFileCmdTest_test_RemFileCmd_1_2_tmp"
saveInFile(tmpStatFile, filterDbStatus1(status.dumpBuf(0, 1, 1)))
self.checkFilesEq(refStatFile, tmpStatFile,
"Incorrect status for REMFILE Command/execution")
def test_RemDiskCmd_2(self):
"""
Synopsis:
Missing file copies, REMDISK Command rejected.
Description:
If there are not enough copies of files stored on a disk requested
to be REMDISK'ed, the server should reject the request indicating
the problem in the response.
If an Email Notification List has been specified, the files in
question, should be reported via email.
Expected Result:
The REMDISK Command should be rejected by the server, which should
detect that there are files on the disk requested to be REMDISK'ed,
which are not available in the system in 3 independent copies.
Test Steps:
- Start server.
- Archive file.
- Submit REMDISK to remove Main Disk hosting the archived file
(execute=0).
- Check on the response that the command is rejected by the server.
- Submit REMDISK to remove Main Disk hosting the archived file
(execute=1).
- Check on the response that the command is rejected by the server.
Remarks:
...
"""
self.prepExtSrv()
client = sendPclCmd()
client.archive("src/SmallFile.fits")
# Remove the cloned disk (execute=0), should fail.
diskId = "tmp-ngamsTest-NGAS-FitsStorage1-Main-1"
status = client.remDisk(diskId, 0)
refStatFile = "ref/ngamsRemDiskCmdTest_test_RemDiskDisk_2_1_ref"
tmpStatFile = "tmp/ngamsRemDiskCmdTest_test_RemDiskDisk_2_1_tmp"
saveInFile(tmpStatFile, filterDbStatus1(status.dumpBuf(0, 1, 1)))
self.checkFilesEq(refStatFile, tmpStatFile,
"Incorrect status for REMDISK Command/no execution")
# Remove the cloned disk (execute=1), should fail.
status = client.remDisk(diskId, 1)
refStatFile = "ref/ngamsRemDiskCmdTest_test_RemDiskDisk_2_2_ref"
tmpStatFile = "tmp/ngamsRemDiskCmdTest_test_RemDiskDisk_2_2_tmp"
saveInFile(tmpStatFile, filterDbStatus1(status.dumpBuf(0, 1, 1)))
self.checkFilesEq(refStatFile, tmpStatFile,
"Incorrect status for REMDISK Command/execution")
def test_ServerLoad_1(self):
"""
Synopsis:
Test that the NG/AMS Server is loading configuration from DB and
initializing as expected.
Description:
Test that the NG/AMS Server can load the configuration properly from
the DB and initialize accordingly.
Expected Result:
The server should load the DB parameters according the specified
NGAS Configuration ID specified and should start up properly.
Test Steps:
- Load the configuration into the DB.
- Start an NG/AMS Server specifying an XML document which only
defines the DB connection + giving the reference to the DB Cfg. ID.
- Verify that the NG/AMS Server starts up properly.
Remarks:
...
"""
cfgName = "test_ServerLoad_1"
cfgObj, _ = self.loadCfg(cfgName)
cfgFile = saveInFile(None, str(cfgObj.genXmlDoc()))
self.prepExtSrv(cfgFile=cfgFile, dbCfgName=cfgName, clearDb=0)
# Archive a file, should be OK.
statObj = sendPclCmd().archive("src/SmallFile.fits")
refStatFile = "ref/ngamsConfigHandlingTest_test_ServerLoad_1_1_ref"
tmpStatFile = saveInFile(None, filterDbStatus1(statObj.dumpBuf()))
self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect status " +\
"returned for Archive Push Request")
def test_RegisterCmd_1(self):
"""
Synopsis:
REGISTER Command/register single file compl. path.
Description:
Test handling of the REGISTER Command under normal circumstances.
Expected Result:
The REGISTER Command should be accepted by the server and should
be executed successfully.
Test Steps:
- Start server.
- Copy file onto NGAS Disk.
- Submit REGISTER Command requesting to register the file copied over
(wait=1).
- Check response from the server that the request was successfully
executed.
- Check the DB info for the registered file.
Remarks:
...
"""
_, dbObj = self.prepExtSrv()
srcFile = "src/SmallFile.fits"
tmpSrcFile = "/tmp/ngamsTest/NGAS/" +\
"FitsStorage2-Main-3/saf/test/SmallFile.fits"
checkCreatePath(os.path.dirname(tmpSrcFile))
shutil.copy(srcFile, tmpSrcFile)
tmpStatFile = sendExtCmd(8888, NGAMS_REGISTER_CMD,
[["path", tmpSrcFile]])
refStatFile = "ref/ngamsRegisterCmdTest_test_RegisterCmd_1_ref"
self.checkFilesEq(refStatFile, tmpStatFile,
"Incorrect status returned for REGISTER command")
diskId = "tmp-ngamsTest-NGAS-FitsStorage2-Main-3"
filePrefix = "ngamsRegisterCmdTest_test_RegisterCmd_1"
fileInfoRef = "ref/" + filePrefix + "_FileInfo_ref"
fileInfoTmp = "tmp/" + filePrefix + "_FileInfo_tmp"
fileId = "TEST.2001-05-08T15:25:00.123"
startTime = time.time()
host_id = getHostName() + ":8888"
while ((time.time() - startTime) < 10):
tmpFileRes = dbObj.getFileInfoFromFileIdHostId(
host_id, fileId, 1, diskId)
if (tmpFileRes): break
if not tmpFileRes:
self.fail(
"Couldn't get fileInfo result from database within 10 seconds")
tmpFileObj = ngamsFileInfo.ngamsFileInfo().unpackSqlResult(tmpFileRes)
saveInFile(fileInfoTmp, filterDbStatus1(tmpFileObj.dumpBuf()))
self.checkFilesEq(fileInfoRef, fileInfoTmp,
"Incorrect info in DB for registered file")
def test_ServerLoad_3(self):
"""
Synopsis:
Test loading of specific configuration from DB.
Description:
Test that the NG/AMS Server is loading configuration from DB and
initializing as expected without mixing up parameters from other
configurations in the DB.
Expected Result:
The NG/AMS Server should load only parameters for the specified
configuration Group ID and ignore other parameters in the DB.
Test Steps:
- Prepare two XML configurations and load them in the DB. Difference
is that all attribute values of the second are set to value=0, which
means that the server could not operate if mixing up parameters from
the two.
- Start server.
- Archive file and check that the file was successfully archived.
Remarks:
...
"""
cfgName1 = "test_ServerLoad_1"
cfgObj, _ = self.loadCfg(cfgName1)
cfgFile = saveInFile(None, cfgObj.genXmlDoc(0))
# For the second cfg. set all the DB Cfg. Group ID to a common value
# + set all values to a non-sense value.
cfgName2 = "test_ServerLoad_2"
tmpCfg = db_aware_cfg("src/ngamsCfg.xml")
for cfgKey in tmpCfg._getXmlDic().keys():
if ((cfgKey[-1] != "]") and (cfgKey.find("Db[1]") == -1)):
tmpCfg.storeVal(cfgKey, "0")
tmpCfg.storeVal("NgamsCfg.Log[1].LocalLogFile",
"/tmp/ngamsTest/NGAS/log/LogFile.nglog", "0")
tmpCfgFile = saveInFile(None, tmpCfg.genXmlDoc(0))
self.loadCfg(cfgName2,
cfgFile=tmpCfgFile,
checkCfg=0,
delDbTbls=0,
dbCfgGroupIds=[0],
createDatabase=False)
# Start server specifying 1st DB cfg.
self.prepExtSrv(cfgFile=cfgFile, dbCfgName=cfgName1, clearDb=0)
statObj = sendPclCmd().archive("src/SmallFile.fits")
refStatFile = "ref/ngamsConfigHandlingTest_test_ServerLoad_3_1_ref"
tmpStatFile = saveInFile(None, filterDbStatus1(statObj.dumpBuf()))
self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect status " +\
"returned for Archive Push Request")
def test_RetrieveCmd_3(self):
"""
Synopsis:
Retrieve file from NGAS Cluster sub-node.
Description:
Test that a file stored on a sub-node is located and returned to the
requestor by the contacted server acting as proxy is Proxy Mode is
eneabled.
Expected Result:
The contacted server should locate the file, forward the request to
the node hosting the file, and should send back the file to the
requestor.
Test Steps:
- Start simulated cluster with master and sub-node.
- Archive file onto sub-node.
- Submit Retrieve Request to Master Node to retrieved the archived
file.
- Check the response from the Master Node.
- Check that the file has been retrieved as expected.
Remarks:
...
"""
self.prepCluster((8000, 8011))
# Archive file into sub-node (port=8011).
sendPclCmd(port=8011).archive("src/TinyTestFile.fits")
# Retrieve a file.
trgFile = "tmp/test_RetrieveCmd_3_1_tmp"
client = sendPclCmd(port=8000)
status = client.retrieve("NCU.2003-11-11T11:11:11.111",
targetFile=trgFile)
# Check reply.
refStatFile = "ref/ngamsRetrieveCmdTest_test_RetrieveCmd_3_1_ref"
tmpStatFile = "tmp/ngamsRetrieveCmdTest_test_RetrieveCmd_3_1_tmp"
saveInFile(tmpStatFile, filterDbStatus1(status.dumpBuf(0, 1, 1)))
self.checkFilesEq(refStatFile, tmpStatFile,
"Incorrect status for RETRIEVE Command/Cluster " +\
"Retrieval")
outFilePath = 'tmp/test_RetrieveCmd_3_1_tmp_unzip'
with gzip.open(trgFile, 'rb') as gz, open(outFilePath, 'wb') as out:
for data in iter(partial(gz.read, 1024), ''):
out.write(data)
# Check the retrieved file (checksum).
refFile = "src/TinyTestFile.fits"
self.checkFilesEq(outFilePath, refFile, "Retrieved file incorrect")
def test_RetrieveCmd_1(self):
"""
Synopsis:
Retrieve file from server hosting the file.
Description:
The purpose of the test is to test the case where it is attempted
to retrieve an archived file directly from the unit hosting the file.
Expected Result:
The server should locate the file and send it back to the requestor.
Test Steps:
- Start server.
- Archive FITS file.
- Retrieve the file and store it in a local file.
- Check reply from the server indiates that the request was handled
successfully.
- Check that the file on disk has been successfully retrieved.
Remarks:
...
"""
self.prepExtSrv()
client = sendPclCmd()
client.archive("src/SmallFile.fits")
# Retrieve the file.
trgFile = "tmp/test_RetrieveCmd_1_1_tmp"
outFilePath = "tmp/SmallFile.fits"
status = client.retrieve("TEST.2001-05-08T15:25:00.123",
targetFile=trgFile)
unzip(trgFile, outFilePath)
# Check reply.
refStatFile = "ref/ngamsRetrieveCmdTest_test_RetrieveCmd_1_1_ref"
tmpStatFile = "tmp/ngamsRetrieveCmdTest_test_RetrieveCmd_1_1_tmp"
saveInFile(tmpStatFile, filterDbStatus1(status.dumpBuf(0, 1, 1)))
self.checkFilesEq(refStatFile, tmpStatFile,
"Incorrect status for RETRIEVE Command/Normal " +\
"Execution")
# Check file retrieved.
self.checkFilesEq("src/SmallFile.fits", outFilePath,
"Retrieved file incorrect")
def test_ServerLoad_2(self):
"""
Synopsis:
Test that the NG/AMS Server is loading configuration from DB and
initializing as expected.
Description:
The purpose of this Test Case is to verify that the NG/AMS Server
can be initialized properly by loading the configuration from the
NGAS DB. The parameter disabling handling of Archive Requests is
set to 0. It is checked that the server after initialization,
rejects Archive Requests.
Expected Result:
The NG/AMS Server should initialize and select disabling of handling
of Archive Requests. Submitting Archive Requests to the server
should result in a rejection of the request.
Test Steps:
- Create an NG/AMS XML Configuration where handling of Archive Requests
is disabled.
- Load this into the DB.
- Start instance of the NG/AMS Server specifying to use the previously
loaded configuration.
- Issue an Archive Request and verify that it is rejected.
Remarks:
...
"""
cfgName = "test_ServerLoad_2"
tmpCfg = db_aware_cfg("src/ngamsCfg.xml").\
storeVal("NgamsCfg.Permissions[1].AllowArchiveReq", "0",
"Permissions-Test")
cfgFile = saveInFile(None, tmpCfg.genXmlDoc(0))
cfgObj, _ = self.loadCfg(cfgName, cfgFile=cfgFile)
cfgFile = saveInFile(None, str(cfgObj.genXmlDoc()))
self.prepExtSrv(cfgFile=cfgFile, dbCfgName=cfgName, clearDb=0)
# Archive a file, should be rejected.
statObj = sendPclCmd().archive("src/SmallFile.fits")
refStatFile = "ref/ngamsConfigHandlingTest_test_ServerLoad_2_1_ref"
tmpStatFile = saveInFile(None, filterDbStatus1(statObj.dumpBuf()))
self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect status " +\
"returned for Archive Push Request")
def test_RetrieveCmd_2(self):
"""
Synopsis:
Attempt to retrieve non-existing file (wrong version).
Description:
Test the case where it is attempted to retrieve a non-exiting file.
An appropriate error response should be generated.
Expected Result:
The server should identify that the file is not found in the NGAS
Archive and should return a response indicating this.
Test Steps:
- Start server.
- Archive FITS file.
- Retrieve file with same File ID as archived file but with another
version than the one allocated to the file.
- Check the error response from the server.
Remarks:
...
"""
self.prepExtSrv()
client = sendPclCmd()
client.archive("src/SmallFile.fits")
# Retrieve the file.
trgFile = "tmp/test_RetrieveCmd_1_1_tmp"
status = client.retrieve("TEST.2001-05-08T15:25:00.123",
fileVersion=2,
targetFile=trgFile)
# Check reply.
refStatFile = "ref/ngamsRetrieveCmdTest_test_RetrieveCmd_2_1_ref"
tmpStatFile = "tmp/ngamsRetrieveCmdTest_test_RetrieveCmd_2_1_tmp"
saveInFile(tmpStatFile, filterDbStatus1(status.dumpBuf(0, 1, 1)))
self.checkFilesEq(refStatFile, tmpStatFile,
"Incorrect status for RETRIEVE Command/Normal " +\
"Execution")
def test_FileVerHandling_01(self):
"""
Synopsis:
Test that REMFILE removes proper File Version.
Description:
The purpose of the test is to verify that if several versions of a
file are found in the archive, the proper version is deleted.
Expected Result:
The chosen version of the file, should be selected and should be
removed.
Test Steps:
- Start server.
- Archive a file 3 times into the archive.
- Clone the file onto another disk.
- Remove version 2 of the cloned file.
Remarks:
...
Test Data:
...
"""
_, dbObj = self.prepExtSrv(cfgProps=(('NgamsCfg.Db[1].Snapshot',
"0"), ))
client = sendPclCmd()
for n in range(3):
stat = client.archive("src/SmallFile.fits")
self.assertEquals('SUCCESS', stat.getStatus())
diskId1 = "tmp-ngamsTest-NGAS-FitsStorage1-Main-1"
fileId = "TEST.2001-05-08T15:25:00.123"
fileVer = 2
status = client.get_status(NGAMS_CLONE_CMD,
pars=[["disk_id", diskId1]])
self.assertEquals('SUCCESS', status.getStatus())
diskId2 = "tmp-ngamsTest-NGAS-FitsStorage2-Main-3"
filePath = "/tmp/ngamsTest/NGAS/FitsStorage2-Main-3/saf/" +\
"2001-05-08/2/TEST.2001-05-08T15:25:00.123.fits.gz"
for execute in [0, 1]:
httpPars = [["disk_id", diskId2], ["file_id", fileId],
["file_version", fileVer], ["execute", execute]]
stat = client.get_status(NGAMS_REMFILE_CMD, pars=httpPars)
self.assertStatus(stat)
fileInfo = ngamsFileInfo.ngamsFileInfo()
try:
fileInfo.read(getHostName(), dbObj, fileId, fileVer, diskId2)
except:
pass
if (execute == 0):
if (not os.path.exists(filePath)):
self.fail("File removed unexpectedly")
else:
if (os.path.exists(filePath)):
self.fail("File not removed as expected")
refStatFile = "ref/ngamsRemFileCmdTest_test_" +\
"FileVerHandling_01_0%d_ref" % (execute + 3)
tmpStatFile = saveInFile(None, filterDbStatus1(fileInfo.dumpBuf()))
self.checkFilesEq(
refStatFile, tmpStatFile,
"Incorrect status for REMFILE Command/execution")
def test_RetrieveCmd_7(self):
"""
Synopsis:
Rerieve file from cluster, Online/Suspended and Offline files
contained in archive.
Description:
The purpose of the test is to verify that the problem that there are
Online and Offline files in the archive and thus the cross-checking
of the number of files for which info was queried from the NGAS DB
detected erroneously a discrepancy.
Expected Result:
The RETRIEVE Command should execute successfully. The check of the
possible files matching the query should not produce and error.
Test Steps:
- Prepare simulated cluster with three nodes. One node should
suspend itself after few seconds.
- Archive the same file onto the three nodes 3 times.
- Make one node exit.
- Wait till the node suspending itself has suspended itself.
- Send RETRIEVE Command to retrieve a file given by its File ID.
- Check that response from the server is OK.
- Send RETRIEVE Command to retrieve a file given by its File ID
and its File Version.
- Check that the response from the server is OK.
- Send RETRIEVE Command to retrieve a file given by the Disk ID,
the File ID and the File Version.
- Check that the response from the server is OK.
Remarks:
...
"""
nodes = [
getHostName() + ":8000",
getHostName() + ":8001",
getHostName() + ":8002"
]
suspPars = [["NgamsCfg.HostSuspension[1].IdleSuspension", "1"],
["NgamsCfg.JanitorThread[1].SuspensionTime", "0T00:00:05"]]
envDic = self.prepCluster((8000, 8001, (8002, suspPars)))
for portNo in [8000, 8001, 8002]:
for n in range(3):
sendPclCmd(port=portNo).archive("src/SmallFile.fits")
sendPclCmd(port=8001).offline()
sendPclCmd(port=8001).exit()
subNode2 = nodes[2]
self.waitTillSuspended(envDic[subNode2][1], subNode2, 45, nodes)
client = sendPclCmd(port=8000)
# Retrieve file (File ID).
fileId = "TEST.2001-05-08T15:25:00.123"
statObj = client.retrieve(fileId)
refStatFile = "ref/ngamsRetrieveCmdTest_test_RetrieveCmd_7_1_ref"
tmpStatFile = saveInFile(None, filterDbStatus1(statObj.dumpBuf()))
self.checkFilesEq(refStatFile, tmpStatFile,
"Unexpected response returned to RETRIEVE Command")
# Retrieve file (File ID + File Version).
statObj = client.retrieve(fileId, fileVersion=2)
tmpStatFile = saveInFile(None, filterDbStatus1(statObj.dumpBuf()))
self.checkFilesEq(refStatFile, tmpStatFile,
"Unexpected response returned to RETRIEVE Command")
def test_CloneCmd_1(self):
"""
Synopsis:
Normal execution of CLONE Command/clone one file/async=1.
Description:
Test normal execution of the CLONE Command whereby async=1.
Expected Result:
An immediate response should be returned indicating that the
CLONE Command has been accepted for execution. The Clone Status
Report should be send out indicating that the file was cloned.
Test Steps:
- Start 1 NG/AMS Server.
- Archive file 2 times.
- Clone one file specifying disk_id, file_id and file_version + async=1.
- Check that the immediate response is correctly returned.
- Wait for the execution of the CLONE Command to finish.
- Check that the Request Info in the NG/AMS Server indicates that the
Clone Request finished.
- Check that the cloned file has arrived on the Target Disk.
- Check that the DB info has been updated as it should.
Remarks:
TODO: Re-implement using _execCloneTest().
"""
srcFile = "src/SmallFile.fits"
cfgObj, dbObj = self.prepExtSrv(cfgProps=(('NgamsCfg.Server[1].RequestDbBackend', 'memory'),))
client = sendPclCmd()
for n in range(2): client.archive(srcFile)
flushEmailQueue()
testUserEmail = getpass.getuser()+"@"+ngamsLib.getCompleteHostName()
statObj = client.get_status(NGAMS_CLONE_CMD,
pars = [["disk_id", srcDiskId],
["file_id", nmuFileId],
["file_version", "1"],
["async", "1"],
["notif_email", testUserEmail]])
refStatFile = "ref/ngamsCloneCmdTest_test_CloneCmd_1_1_ref"
tmpStatFile = saveInFile(None, filterDbStatus1(statObj.dumpBuf()))
self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect status for " +\
"CLONE command/successfull cloning")
finalStatObj = waitReqCompl(client, statObj.getRequestId())
complPer = str(finalStatObj.getCompletionPercent())
self.checkEqual("100.0", complPer,
genErrMsgVals("Incorrect Request Status for CLONE " +\
"Command/Completion Percent", "100.0",
complPer))
if _checkMail:
mailCont = getEmailMsg(["NGAS Host:", "Total proc", "Handling time"])
tmpStatFile = "tmp/ngamsCloneCmdTest_test_CloneCmd_1_tmp"
refStatFile = "ref/ngamsCloneCmdTest_test_CloneCmd_1_ref"
saveInFile(tmpStatFile, mailCont)
self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect/missing " +\
"CLONE Status Notification Email Msg")
tmpFitsFile = "/tmp/ngamsTest/NGAS/FitsStorage2-Main-3/saf/" +\
"2001-05-08/1/TEST.2001-05-08T15:25:00.123.fits.gz"
unzippedTmp = genTmpFilename()
unzip(tmpFitsFile, unzippedTmp)
self.checkFilesEq(srcFile, unzippedTmp, "Incorrect cloned file generated")
diskId = "tmp-ngamsTest-NGAS-FitsStorage2-Main-3"
filePrefix = "ngamsCloneCmdTest_test_CloneCmd_1"
fileInfoRef = "ref/" + filePrefix + "_FileInfo_ref"
fileInfoTmp = "tmp/" + filePrefix + "_FileInfo_tmp"
fileInfo = ngamsFileInfo.\
ngamsFileInfo().read(getHostName() + ":8888",
dbObj, "TEST.2001-05-08T15:25:00.123",
1, diskId)
saveInFile(fileInfoTmp, filterDbStatus1(fileInfo.dumpBuf()))
self.checkFilesEq(fileInfoRef, fileInfoTmp, "Incorrect info in DB " +\
"for cloned file")
def _execCloneTest(testObj,
testData,
refStatFile):
"""
Execute the Clone Command Test based on the given input data. There is
always waited for command execution.
A more thorough test (using async=1) is implemented in test_CloneCmd_1 and
test_CloneCmd_2 (testing also the Email Notification in connection with the
CLONE Command).
testData: List with test information:
[<Disk ID>, <File ID>, <File Ver>, <Trg Disk ID>,
<Subnode (0|1)>] (list)
refStatFile: Name of reference file (string).
Returns: Void.
"""
diskId = testData[0]
fileId = testData[1]
fileVer = testData[2]
trgDisk = testData[3]
subNode = testData[4]
if (subNode):
testObj.prepCluster((8000, 8011))
clNcu = sendPclCmd(port=8011)
else:
testObj.prepExtSrv(port=8000)
clMnu = sendPclCmd(port=8000)
for n in range(5):
statObj = clMnu.archive("src/SmallFile.fits")
if (subNode): clNcu.archive("src/TinyTestFile.fits")
cmdPars = []
if (diskId): cmdPars.append(["disk_id", diskId])
if (fileId): cmdPars.append(["file_id", fileId])
if (fileVer): cmdPars.append(["file_version", fileVer])
if (trgDisk): cmdPars.append(["target_disk_id", trgDisk])
cmdPars.append(["async", "0"])
cmdPars.append(["notif_email", getpass.getuser() + "@" +\
ngamsLib.getCompleteHostName()])
flushEmailQueue()
statObj = clMnu.get_status(NGAMS_CLONE_CMD, pars = cmdPars)
# Check returned status.
tmpStatFile = saveInFile(None, filterDbStatus1(statObj.dumpBuf(0, 1, 1)))
errMsg = "Executed CLONE Command: Disk ID: %s, File ID: %s, " +\
"File Version: %s, Target Disk: %s. Message: %s"
testObj.checkFilesEq(refStatFile + "_1_ref", tmpStatFile, errMsg %
(str(diskId), str(fileId), str(fileVer), str(trgDisk),
str(statObj.getMessage())))
if _checkMail:
# Check Email Notification Message.
mailCont = getEmailMsg(["NGAS Host:", "Total proc", "Handling time"])
mailCont = _sortRepFileList(mailCont)
tmpStatFile = saveInFile(None, mailCont)
saveInFile(tmpStatFile, mailCont)
testObj.checkFilesEq(refStatFile + "_2_ref", tmpStatFile, errMsg %
(str(diskId), str(fileId), str(fileVer),
str(trgDisk), "Illegal CLONE Command Email " +\
"Notification Message"))
def test_NormalOp_1(self):
"""
Synopsis:
Test normal operation of the NG/AMS Archive Client.
Description:
It is tested that a file can be archived via a link in the Archive
Queue.
Expected Result:
After the archiving the link is moved to the Archive Files
Area. It is also checked that the NG/AMS XML Status Document is
created in the Archived Files Area. After the expiration time for
keeping archived files has expired, the archived file and the XML
status document should be deleted.
Test Steps:
- Start NG/AMS Server.
- Start instance of the NG/AMS Archive Client.
- Create a link from a legal test (FITS) file into the Archive Queue.
- Test that the file is archived within 20s and moved to the Archived
Files Area.
- Test that the XML Status Document from NG/AMS is stored in the
Archived Files Area.
- Check that after the given expiration time for the Archived Files
Area, that the archived file + the XML Status Document are removed.
- Stop the Archive Client.
Remarks:
...
"""
self.prepExtSrv()
# Make sure the the queue subdir exist before the launch the client;
# otherwise the client and this test might find themselves in a race
# condition and the test might fail
d = os.path.abspath(os.path.join(arcCliDir, 'queue'))
if not os.path.exists(d):
os.makedirs(d)
self.startArchiveClient()
# Archive a file as copy and link.
# Make sure at least the quee dir is already created
srcFile = os.path.abspath("src/SmallFile.fits")
shutil.copy(srcFile, os.path.join(arcCliDir, 'queue'))
os.symlink(srcFile, os.path.join(arcCliDir, 'queue', 'Test.fits'))
# Check that files are being archived (within 20s) + NG/AMS Status
# Documents created.
file1Pat = arcCliDir + "/archived/*___SmallFile.fits"
file1StatPat = file1Pat + "___STATUS.xml"
file2Pat = arcCliDir + "/archived/*___Test.fits"
file2StatPat = file2Pat + "___STATUS.xml"
startTime = time.time()
filesFound = 0
while ((time.time() - startTime) < 20):
globFile1Pat = glob.glob(file1Pat)
globFile1StatPat = glob.glob(file1StatPat)
globFile2Pat = glob.glob(file2Pat)
globFile2StatPat = glob.glob(file2StatPat)
if ((len(globFile1Pat) == 1) and (len(globFile1StatPat) == 1)
and (len(globFile2Pat) == 1)
and (len(globFile2StatPat) == 1)):
filesFound = 1
break
if (not filesFound):
if (not len(globFile1Pat)):
errMsg = "Did not find status file: " + file1Pat
elif (not len(globFile1StatPat)):
errMsg = "Did not find status XML document: " + file1StatPat
elif (not len(globFile2Pat)):
errMsg = "Did not find status file: " + file2Pat
else:
# (not len(globFile2StatPat)):
errMsg = "Did not find status XML document: " + file2StatPat
self.fail(errMsg)
# Check the contents of one of the status documents.
statObj = ngamsStatus.ngamsStatus().load(globFile1StatPat[0])
refStatFile = "ref/ngamsArchiveClientTest_test_NormalOp_1_1_ref"
tmpStatFile = saveInFile(
None,
filterDbStatus1(statObj.dumpBuf(), [
"BytesStored:", "NumberOfFiles:", "FileName:", "FileVersion:"
]))
self.checkFilesEq(refStatFile, tmpStatFile,
"Incorrect info in Archive Command " +\
"XML Status Document")
# Check that the status documents are removed within 10s.
filesRemoved = 0
startTime = time.time()
while ((time.time() - startTime) < 20):
globFile1Pat = glob.glob(file1Pat)
globFile1StatPat = glob.glob(file1StatPat)
globFile2Pat = glob.glob(file2Pat)
globFile2StatPat = glob.glob(file2StatPat)
if ((len(globFile1Pat) == 0) and (len(globFile1StatPat) == 0)
and (len(globFile2Pat) == 0)
and (len(globFile2StatPat) == 0)):
filesRemoved = 1
break
if (not filesRemoved):
if (len(globFile1Pat)):
errMsg = "Did not remove status file: " + globFile1Pat[0]
elif (len(globFile1StatPat)):
errMsg = "Did not remove status XML document: " +\
globFile1StatPat[0]
elif (len(globFile2Pat)):
errMsg = "Did not remove status file: " + file2Pat[0]
else:
# (len(globFile2StatPat)):
errMsg = "Did not remove status XML document: " + file2StatPat[
0]
self.fail(errMsg)
