def test_WakeUpStatus_1(self): """ Synopsis: Test that suspended server is woken up when STATUS?host_id=<Sub-Node> specifying that node is sent. Description: If a STATUS Command is send to a node requesting status from another node (Proxy Mode), and the target is suspended, the contacted node must wake up the suspended target node. The purpose of this test is to verify this use case. Expected Result: The contacted node should identify the target node as suspended, and should wake it up prior to forwarding the STATUS Command acting as proxy. Test Steps: - Start simulated cluster specifying that one node can suspend itself. - Wait till sub-node has suspended itself. - Send STATUS Command to the simulated master requesting status from the suspended sub-node. - Check the response to the STATUS Command. - Cross check that the suspended node is now woken up by sending a STATUS Command directly to the sub-node. Remarks: ... """ cfgParDic = {"8001": [["%s.IdleSuspensionTime" % SUSP_EL, "5"]]} dbConObj = prepSimCluster(self, cfgParDic=cfgParDic)[masterNode][1] self.waitTillSuspended(dbConObj, subNode1, 30, susp_nodes) # 1. Send STATUS Command to sub-node using master as proxy. statObj = sendPclCmd(port=8000, auth=AUTH).\ get_status(NGAMS_STATUS_CMD, pars = [["host_id", subNode1]]) statBuf = filterOutLines(statObj.dumpBuf(), ["Date:", "Version:"]) tmpStatFile = saveInFile(None, statBuf) refStatFile = "ref/ngamsIdleSuspensionTest_test_WakeUpStatus_1_1_ref" refStatFile = saveInFile(None, loadFile(refStatFile) % getHostName()) self.checkFilesEq(refStatFile, tmpStatFile, "Sub-node not woken up as expected") # 2. Double-check that sub-node is no longer suspended. statObj = sendPclCmd(port=8001, auth=AUTH).status() statBuf = filterOutLines(statObj.dumpBuf(), ["Date:", "Version:"]) tmpStatFile = saveInFile(None, statBuf) self.checkFilesEq(refStatFile, tmpStatFile, "Sub-node not woken up as expected") # Clean up. self.markNodesAsUnsusp(dbConObj, susp_nodes)
def test_ChangeLocLogLev_1(self): """ Synopsis: Change the Local (Log File) Log Level Online. Description: The purpose of this Test Case is to test that the Local Log Level can be changed while the NG/AMS Server is Online. Expected Result: The Log Level should be increased and more logs produced in the Local Log File. Test Steps: - Start normal NG/AMS Server (Log Level=3). - Send a CONFIG Command to the server changing the Log Level to 4. - Check the response to the CONFIG Command is correct. Remarks: TODO: Check in Log File that low level logs are produced. """ self.prepExtSrv() statObj = sendPclCmd().get_status(NGAMS_CONFIG_CMD, pars=[["log_local_log_level", "4"]]) tmpStatFile = "tmp/ngamsConfigCmdTest_test_ChangeLocLogLev_1_1_tmp" refStatFile = "ref/ngamsConfigCmdTest_test_ChangeLocLogLev_1_1_ref" saveInFile( tmpStatFile, filterOutLines(statObj.dumpBuf(), ["Date", "Version", "HostId"])) self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect info in " +\ "CONFIG Command XML Status " + "Document/log_local_log_level")
def test_ArchiveCmd_Err_1(self): """ Synopsis: Issue ARCHIVE Command/request times out. Description: The purpose of the test is to check that a request that times out is handled properly by the C-Client/API and a proper error message is produced. Expected Result: After the given timeout, the C-Client/API should generate a timeout error message. Test Steps: - Start speciel instance of server where Archive Requests blocks. - Issue Archive Request (small file) specifying a timeout of 10s. - Capture the output from the ngamsCClient, filter this, and check that the proper error message has been generated. Remarks: ... """ rmFile("tmp/reqCallBack_tmp") saveInFile("tmp/reqCallBack_tmp", "reqCallBack_BlockCmds1") self.prepExtSrv(srvModule="ngamsSrvTestDynReqCallBack") out = _execCClient(unpackXmlStat=0, pars=[["-port", "8888"], ["-cmd", "ARCHIVE"], ["-fileUri", "src/SmallFile.fits"], ["-timeOut", "5"]]) tmpStatFile = saveInFile(None, filterOutLines(out, ["Host"])) refStatFile = "ref/ngamsCClientTest_test_ArchiveCmd_Err_1_ref" self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect handling " +\ "of timeout of Archive Request in C-Client/API")
def test_ArchiveCmd_Err_4_1(self): """ Synopsis: Correct HTTP response, but illegal NG/AMS XML status document. Description: The purpose of the test is to verify that the C-Client/API handles correctly the situation where an incorrectly formatted XML status response is contained in the response from the server and the proper error code generated by the C-API. Expected Result: The C-API detects the wrongly formatted NG/AMS XML status document, and produces the appropriate error code. Test Steps: - Start special instance of the server which generates an HTTP response with a corrupt XML NG/AMS status document. - Issue an Archive Request via the ngamsCClient. - Compare the output from ngamsCClient to check that the invalid response was correctly handled. Remarks: ... """ # TODO: From V4.0, this test case produces the error: # # Error Code: -103 # Message: Invalid reply from data server # # - and not: # # Error Code: -4 # Message: Problem communicating with server # Status: FAILURE # # - as for previous version. # # This should be investigated and resolved. rmFile("tmp/reqCallBack_tmp") saveInFile("tmp/reqCallBack_tmp", "reqCallBack_IllegalResp") self.prepExtSrv(srvModule="ngamsSrvTestDynReqCallBack") httpResp = "HTTP/1.0 200 OK\015\012" +\ "Server: NGAMS/v2.3/2004-07-12T11:39:39\015\012" +\ "Date: Thu, 7 Oct 2004 16:20:28 GMT\015\012" +\ "Expires: Thu, 7 Oct 2004 16:20:28 GMT\015\012" +\ "Content-Type: text/xml\015\012" +\ "Content-Length: 36\015\012" +\ "\015\012" +\ "COMPLETELY CORRUPT NG/AMS XML STATUS" saveInFile("tmp/ngamsServerTestIllegalResp_tmp", httpResp) out = _execCClient(unpackXmlStat=0, pars=[["-port", "8888"], ["-cmd", "ARCHIVE"], ["-fileUri", "src/SmallFile.fits"]]) tmpStatFile = saveInFile(None, filterOutLines(out, ["Host"])) refStatFile = "ref/ngamsCClientTest_test_ArchiveCmd_Err_4_1_ref" self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect handling " +\ "of corrupt server HTTP response in C-Client/API")
def test_StatusCmd_5(self): """ Synopsis: Issue STATUS Command/disk_id. Description: Test that the C-Client/API can handle STATUS Command with -diskId. Expected Result: The STATUS Command requesting for info about a specified disk, should be properly transferred via the C-Client/API and executed on the server. Test Steps: - Start server. - Issue STATUS Command requesting for info about a certain disk. - Compare the returned XML disk info with a reference file. Remarks: ... """ cfgObj, dbObj = self.prepExtSrv(port=8111) diskId = "tmp-ngamsTest-NGAS-FitsStorage1-Main-1" statObj = _execCClient(pars=[["-cmd", "STATUS"], ["-diskId", diskId], ["-port", "8111"]])[0] tmpStatFile = "tmp/ngamsCClientTest_test_StatusCmd_5_1_tmp" refStatFile = "ref/ngamsCClientTest_test_StatusCmd_5_1_ref" saveInFile(tmpStatFile, filterOutLines(statObj.dumpBuf(), STD_DISK_STAT_FILT)) self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect info in " +\ "STATUS Command XML Status Document/disk_id")
def test_StatusCmd_1(self): """ Synopsis: Issue STATUS Command/basic (no parameters). Description: Issue a STATUS Command via the C-Client (on the shell). Expected Result: The STATUS Command should be accepted and executed by the C-Client and the server. Test Steps: - Start standard server. - Execute the ngamsCClient tool on the shell. - Capture the output from the ngamsCClient and compare this with the expected output. Remarks: ... """ cfgObj, dbObj = self.prepExtSrv(port=8000) statObj = _execCClient(pars=[["-port", "8000"], ["-cmd", "STATUS"]])[0] refStatFile = "ref/ngamsCClientTest_test_StatusCmd_1_1_ref" refStatFile = saveInFile( None, loadFile(refStatFile) % (getHostName() + ":8000")) tmpStatFile = saveInFile( None, filterOutLines(statObj.dumpBuf(), ["Date", "Version"])) self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect info in " +\ "STATUS Command XML Status Document")
def test_ArchiveCmd_Err_3_1(self): """ Synopsis: Handling of corrupted HTTP response. Description: The purpose of the test is test that a corrupted HTTP response is properly handled by the C-Client/API. The response is corrupted such that it only contains a '\015\012' ({Slash r}{Slash n}). Expected Result: The corrupt HTTP response cannot be unpacked/interpreted as an XML document. This should be detected by the C-API and a proper error message returned. Test Steps: - Start special instance of the server class, which produces an illegal response. - Issue an ARCHIVE Command via the ngamsCClient. - Check that the output produced on stdout refers to the appropriate error message. Remarks: ... """ # TODO: From V4.0, this test case produces the error: # # Error Code: -4 # Message: Problem communicating with server # # - and not: # # Error Code: -103 # Message: Invalid reply from data server # # - as for previous version. # # This should be investigated and resolved. rmFile("tmp/reqCallBack_tmp") saveInFile("tmp/reqCallBack_tmp", "reqCallBack_IllegalResp") rmFile("tmp/ngamsServerTestIllegalResp_tmp") saveInFile("tmp/ngamsServerTestIllegalResp_tmp", "\015\012") self.prepExtSrv(srvModule="ngamsSrvTestDynReqCallBack") out = _execCClient(unpackXmlStat=0, pars=[["-port", "8888"], ["-cmd", "ARCHIVE"], ["-fileUri", "src/SmallFile.fits"]]) tmpStatFile = saveInFile(None, filterOutLines(out, ["Host"])) refStatFile = "ref/ngamsCClientTest_test_ArchiveCmd_Err_3_1_ref" self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect handling " +\ "of corrupt server HTTP response in C-Client/API")
def test_WakeUpStatus_2(self): """ Synopsis: Test that suspended server is woken up when STATUS?file_access is issued, specifying a file on a suspended sub-node. Description: The purpose of the test is to verify that a suspended sub-node is woken up by a master node acting as proxy if a STATUS?file_id is send pointing to a file on the suspended sub-node. Expected Result: The suspended sub-node should be woken up by the master node which after the suspended sub-node has come Online should forward the request which is executed on the sub-node and the result send back to the client through the master node. Test Steps: - Prepare simulated cluster with unit, which suspends itself. - Archive 3 files onto sub-node. - Wait till sub-node has suspended itself. - Send a STATUS?file_id request to the Master Node specifying one of the files previously archived. - Check the result returned from the sub-node via the Master Node. Remarks: ... """ cfgParDic = {"8001": [["%s.IdleSuspensionTime" % SUSP_EL, "5"]]} dbConObj = prepSimCluster(self, cfgParDic=cfgParDic)[masterNode][1] # Archive some files on the two nodes and wait till sub-node # has suspended itself. sendPclCmd(port=8000, auth=AUTH).archive("src/TinyTestFile.fits") sendPclCmd(port=8000, auth=AUTH).archive("src/SmallFile.fits") sendPclCmd(port=8001, auth=AUTH).archive("src/SmallFile.fits") self.waitTillSuspended(dbConObj, subNode1, 20, susp_nodes) # Retrieve information about the file on the suspended sub-node. fileId = "TEST.2001-05-08T15:25:00.123" statObj = sendPclCmd(port=8000, auth=AUTH).\ get_status(NGAMS_STATUS_CMD, pars=[["file_access", fileId]]) statBuf = filterOutLines(statObj.dumpBuf(), ["Date:", "Version:"]) tmpStatFile = saveInFile(None, statBuf) refStatFile = "ref/ngamsIdleSuspensionTest_test_WakeUpStatus_2_1_ref" refStatFile = saveInFile(None, loadFile(refStatFile) %\ (getHostName(), getHostName())) self.checkFilesEq(refStatFile, tmpStatFile, "Unexpected reply to STATUS?file_access request")
def test_RetrieveCmd_Err_1_2(self): """ Synopsis: Issue RETRIEVE Command/server dies during initial handling. Description: Check that the situation where the server dies during the initial handling of a Retrieve Request is correctly handled by the C-API. Expected Result: The C-API should detect that the server died (=broken socket connection) and should produce the appropriate error code, which is printed on stdout by the ngamsCClient. Test Steps: - Start a special instance of the server, which kills itself when receiving a RETRIEVE Command. - Archive a file. - Issue a RETRIEVE Command to retrieve the archived file. - Verify that the proper output is produced by ngamsCClient indicating the problem. Remarks: ... """ rmFile("tmp/reqCallBack_tmp") saveInFile("tmp/reqCallBack_tmp", "reqCallBack_SrvCrash2") self.prepExtSrv(srvModule="ngamsSrvTestDynReqCallBack") out =\ _execCClient(unpackXmlStat = 0, pars = [["-port", "8888"], ["-cmd", "RETRIEVE"], ["-fileId", "TEST.2001-05-08T15:25:00.123"], ["-timeOut", "10"]]) tmpStatFile = saveInFile(None, filterOutLines(out, ["Host"])) refStatFile = "ref/ngamsCClientTest_test_RetrieveCmd_Err_1_2_ref" self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect handling " +\ "of timeout of Retrieve Request in C-Client/API")
def test_RetrieveCmd_Err_1_3(self): """ Synopsis: Issue RETRIEVE Command/server dies (connection broken) while the server is sending the data across. Description: The purpose of the test is to verify that the C-Client/API handle properly the situation where the socket connection where the server is writing data to the client breaks. Expected Result: The C-API should detect the problem, and that the file has been received incomplete and should return the proper error code, which is written by ngamsCClient on stdout. Test Steps: - Start special instance of the server class, which terminates itself while returned requested data. - Archive a file. - Retrieve the file. - Check that the proper error response is created on stdout by ngamsCClient. Remarks: ... """ self.prepExtSrv(srvModule="ngamsSrvTestBrokSockRetrieve") sendPclCmd().archive("src/SmallFile.fits") out = _execCClient(unpackXmlStat=0, pars=[["-port", "8888"], ["-cmd", "RETRIEVE"], ["-fileId", "TEST.2001-05-08T15:25:00.123"], ["-timeOut", "5"]]) tmpStatFile = saveInFile(None, filterOutLines(out, ["Host"])) refStatFile = "ref/ngamsCClientTest_test_RetrieveCmd_Err_1_3" self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect handling " +\ "of broken read socket in C-Client/API")
def test_RetrieveCmd_Err_1_1(self): """ Synopsis: Issue RETRIEVE Command/request times out. Description: The purpose of the test is to verify that the situation where a Retrieve Request times out is correctly handled by the C-API. Expected Result: After the specified timeout is reached, the appropriate error code is returned to the client. Test Steps: - Start special instance of the server, which blocks on a Retrieve Request (no response send). - Issue a Retrieve Request via the ngamsCClient with timeout 10s. - Verify that after 10s the correct error code is returned by the C-API (printed by ngamsCClient on stdout). Remarks: ... """ rmFile("tmp/reqCallBack_tmp") saveInFile("tmp/reqCallBack_tmp", "reqCallBack_AccArchiveBlock2") self.prepExtSrv(srvModule="ngamsSrvTestDynReqCallBack") sendPclCmd(port=8888).archive("src/SmallFile.fits") out =\ _execCClient(unpackXmlStat = 0, pars = [["-port", "8888"], ["-cmd", "RETRIEVE"], ["-fileId", "TEST.2001-05-08T15:25:00.123"], ["-timeOut", "10"]]) tmpStatFile = saveInFile(None, filterOutLines(out, ["Host"])) refStatFile = "ref/ngamsCClientTest_test_RetrieveCmd_Err_1_1_ref" self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect handling " +\ "of timeout of Retrieve Request in C-Client/API")
def test_ArchiveCmd_Err_5_1(self): """ Synopsis: Issue ARCHIVE Command/socket breaks while writing data on it. Description: The purpose of the test is to verify that the C-API handles properly the situation where the socket connection breaks while data is being written on it (during an Archive Push Request). Expected Result: The C-API should detect that the write socket connection breaks, and should produce the appropriate error message. Test Steps: - Start special instance of server which terminates itself while the client writes the data to the server. - Issue Archive Push Request with a big file to the server. - Verify that the proper error response is produced by the C-API. Remarks: ... """ rmFile("tmp/reqCallBack_tmp") saveInFile("tmp/reqCallBack_tmp", "reqCallBack_SrvCrash1") self.prepExtSrv(srvModule="ngamsSrvTestDynReqCallBack") cpFile("src/WFI-TEST.fits.Z", "tmp/WFI-TEST_tmp.fits.Z") rmFile("tmp/WFI-TEST_tmp.fits") subprocess.check_call(['uncompress', 'tmp/WFI-TEST_tmp.fits.Z']) out = _execCClient(unpackXmlStat=0, pars=[["-port", "8888"], ["-cmd", "ARCHIVE"], ["-fileUri", "tmp/WFI-TEST_tmp.fits"]]) tmpStatFile = saveInFile(None, filterOutLines(out, ["Host"])) refStatFile = "ref/ngamsCClientTest_test_ArchiveCmd_Err_5_1_ref" self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect handling " +\ "of broken write socket in C-Client/API")
def test_ArchiveCmd_Err_3_3(self): """ Synopsis: Issue ARCHIVE Command/server sends back a nonsense HTTP response. Description: The purpose of the test is test that a corrupted HTTP response is properly handled by the C-Client/API. The response is corrupted such that it contains 'noise'. Expected Result: The C-API should detect the illegally formatted response and return the correct error code. Test Steps: - Start special instance of the server that generates a non-sense HTTP response. - Issue an Archive Request via the ngamsCClient. - Check that the proper error message is produced by the ngamsCClient Remarks: ... """ rmFile("tmp/reqCallBack_tmp") saveInFile("tmp/reqCallBack_tmp", "reqCallBack_IllegalResp") rmFile("tmp/ngamsServerTestIllegalResp_tmp") saveInFile("tmp/ngamsServerTestIllegalResp_tmp", "f-423hcqfe-0") self.prepExtSrv(srvModule="ngamsSrvTestDynReqCallBack") out = _execCClient(unpackXmlStat=0, pars=[["-port", "8888"], ["-cmd", "ARCHIVE"], ["-fileUri", "src/SmallFile.fits"]]) tmpStatFile = saveInFile(None, filterOutLines(out, ["Host"])) refStatFile = "ref/ngamsCClientTest_test_ArchiveCmd_Err_3_3_ref" self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect handling " +\ "of corrupt server HTTP response in C-Client/API")
def test_ArchiveCmd_Err_2(self): """ Synopsis: Issue ARCHIVE Command/server crashes (broken socket connection). Description: The purpose of the test is to verify the correct handling/behavior of the C-Client/API in the case the socket connection to the server breaks. Expected Result: During the request handling the socket connection is provoked to break and the C-Client/API should detect this and shoudl produce the proper error message. Test Steps: - Start special instance of the server which makes itself crash when a request is received (to simulate a broken socket connection). - Submit an ARCHIVE Command via the C-Client/API (small file). - Check the result produced by the ngamsCClient on stdout that this is as expected. Remarks: ... """ rmFile("tmp/reqCallBack_tmp") saveInFile("tmp/reqCallBack_tmp", "reqCallBack_SrvCrash1") self.prepExtSrv(srvModule="ngamsSrvTestDynReqCallBack") out = _execCClient(unpackXmlStat=0, pars=[["-port", "8888"], ["-cmd", "ARCHIVE"], ["-fileUri", "src/SmallFile.fits"]]) tmpStatFile = saveInFile(None, filterOutLines(out, ["Host"])) refStatFile = "ref/ngamsCClientTest_test_ArchiveCmd_Err_2_ref" self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect handling " +\ "of crash of server in C-Client/API")
def test_WakeUpCheckfile_1(self): """ Synopsis: Check that CHECKFILE?file_id&file_version correctly handled when the referenced file is stored on a suspended node. Description: The purpose of the test is to verify that a CHECKFILE Command is properly handled by a Master node acting as a proxy and interacting with a suspended sub-node. Expected Result: The contacted Master Node should identify that the sub-node hosting the specified file is suspended, and should wake it up before forwarding the request. Subsequently it should forward the request to the sub-node and send back the result to the client. Test Steps: - Start simulated cluster with a sub-node suspending itself after a short while. - Archive 2 FITS file 2 times each. - Wait till the sub-node suspends itself. - Send a CHECKFILE Command to the Master node specifying to check one of the previously archived files. - Check response from the sub-node. - Check from the log entries in the Master Node that the CHECKFILE Command has been successfully executed. - Check from the log entries on the sub-node that the CHECKFILE Command has been successfully executed. Remarks: ... """ cfgParDic = {"8001": [["%s.IdleSuspensionTime" % SUSP_EL, "5"]]} dbConObj = prepSimCluster(self, cfgParDic=cfgParDic)[masterNode][1] sendPclCmd(port=8001, auth=AUTH).archive("src/TinyTestFile.fits") sendPclCmd(port=8001, auth=AUTH).archive("src/SmallFile.fits") sendPclCmd(port=8000, auth=AUTH).archive("src/TinyTestFile.fits") sendPclCmd(port=8000, auth=AUTH).archive("src/SmallFile.fits") self.waitTillSuspended(dbConObj, subNode1, 10, susp_nodes) # Execute CHECKFILE Command on a file on the suspended sub-node. file_id = "TEST.2001-05-08T15:25:00.123" cmdPars = [["file_id", file_id], ["file_version", "1"]] statObj = sendPclCmd(port=8000, auth=AUTH).\ get_status(NGAMS_CHECKFILE_CMD, pars=cmdPars) # Check that request response is as expected. statBuf = filterOutLines(statObj.dumpBuf(), ["Date:", "Version:"]) tmpStatFile = saveInFile(None, statBuf) refStatFile = "ref/ngamsIdleSuspensionTest_test_WakeUpCheckfile_1_1_ref" refStatFile = saveInFile(None, loadFile(refStatFile) %\ (subNode1, subNode1)) self.checkFilesEq(refStatFile, tmpStatFile,"CHECKFILE Command not " +\ "executed on sub-node as expected") #tstStr = "NGAMS_INFO_FILE_OK:4056:INFO: Checked file with File ID: "+\ # "TEST.2001-05-08T15:25:00.123/File Version: 1/Disk ID" #if (statObj.getMessage().find(tstStr) == -1): #self.fail("CHECKFILE Command not executed on sub-node as expected") # Check that expected log entries found in the Master Node Log File. refStatFile = "ref/ngamsIdleSuspensionTest_test_WakeUpCheckfile_1_2_ref" testTags = loadFile(refStatFile) % (subNode1, ) sendPclCmd(port=8000, auth=AUTH).status() masterLogBuf = loadFile(masterNodeLog) self.checkTags(masterLogBuf, testTags, showBuf=0) # Check that expected log entries found in the Sub-Node Log File. tmpTag = "File list to check: (1: Location:LOCAL, Host:%s, " +\ "Version:1) (2: Location:LOCAL, Host:%s, Version:1)" testTags = [('CHECKFILE?file_id=' + urllib.quote(file_id) + '&file_version=1&time_out=60.0'), tmpTag % (subNode1, subNode1)] subNodeLogBuf = loadFile(subNode1Log) self.checkTags(subNodeLogBuf, testTags, showBuf=0)