def archiveThread(testObj,
no,
inc,
dummy):
"""
Archive a file X times.
testObj: Reference to instance of ngamsTestSuite object (ngamsTestSuite).
no: Number allocated to thread (integer).
inc: Increment ARCFILE keyword before submitting each Archive Request
(0|1/integer).
Returns: Void
"""
if (inc):
filename = "tmp/ngamsArchiveStressTest_%d.fits" % no
cpFile("src/TinyTestFile.fits", filename)
incArcfile(filename, step=(100 * no))
else:
filename = "src/TinyTestFile.fits"
testStr = TST_STR1 % os.path.basename(filename)
for n in range(5):
if (not RUN_TEST):
THREAD_STAT[no] = "STOPPED"
break
if (inc): incArcfile(filename)
statObj = sendPclCmd(auth=AUTH).archive(filename)
if (statObj.getMessage() != testStr):
THREAD_STAT[no] = "FAILURE: Archive Request failed"
THREAD_STAT[no] = "SUCCESS"
def test_DbSnapshot_6(self):
"""
Synopsis:
Test that DB Snapshot is updated correctly when file are registered.
Description:
The purpose of the test is to verify that the DB Snapshot of a disk
on which files are being registered, is properly updated according
to the files registered.
Expected Result:
After a short while, the information for the files registered should
appear in the DB Snapshot of the disk.
Test Steps:
- Start server.
- Copy over a small test FITS file in 3 copies.
- Send a REGISTER Command to register the files copied over
(specifying root directory of the files).
- Check that the DB Snapshot is updated within a given period of time.
Remarks:
...
"""
cfgObj, dbObj = _prepSrv(self)
client = sendPclCmd()
regTestDir = "/tmp/ngamsTest/NGAS/FitsStorage1-Main-1/reg_test"
checkCreatePath(regTestDir)
for n in range(3):
cpFile('src/SmallFile.fits', os.path.join(regTestDir, "%d.fits" % n))
client.get_status(NGAMS_REGISTER_CMD, pars = [["path", regTestDir]])
_checkContDbSnapshot(self, 6, ["FitsStorage1-Main-1"], 1, 1)
def test_unfinished_archive_goes_to_queue(self):
"""An uncompleted archiving operation should go back to the queue dir"""
# We start a server that takes 5 seconds to do ARCHIVEs
save_to_tmp("handleHttpRequest_Block5secs", fname="handleHttpRequest_tmp")
self.start_srv(srvModule="test.support.ngamsSrvTestDynReqCallBack")
self.create_monitor(fs_poll_period=0.1, archive_poll_period=0.1)
queued_fname = os.path.join(self.monitor.queue_dir, 'test.fits')
cpFile(self.resource('src/SmallFile.fits'), queued_fname)
self.assert_monitor_files(archiving=1)
self.terminateAllServer()
self.monitor.stop()
self.assert_monitor_files(queued=1)
def test_NormalExec_2(self):
"""
Synopsis:
File existing, remove via path, +/-execute.
Description:
The purpose of the test is to verify the normal functioning of the
DISCARD Command when DISCARD'ing files referred to by their path.
It is tried both to execute the command with execute=0 and execute=1.
The file is not registered in the NGAS DB.
This simulates the situation where spurious files are removed from the
NGAS system only referred to by their path.
Expected Result:
When the DISCARD Command is issued, execute=0, the NG/AMS Server should
find the file and return a response indicating that the file is only
available in 1 copy. When submitting the DISCARD Command with
execute=1, the file should disappear from the disk.
Test Steps:
- Start NG/AMS Server.
- Copy a test file onto a disk of the NGAS System.
- Issue a DISCARD Command specifying the copied file (execute=0).
- Check that the response indicates that the DISCARD Request is
granted.
- Reissue the DISCARD Command specifying the copied file (execute=1).
- Check that the response indicates that the file has been removed.
- Check that the file has disappeared from the disk.
Remarks:
TODO!: Implement check to verify that file has been removed from the
disk.
"""
self.prepExtSrv()
trgFile = "/tmp/ngamsTest/NGAS/FitsStorage3-Main-5/saf/SmallFile.fits"
cpFile(srcFitsFile, trgFile)
tmpStatFile = sendExtCmd(8888, NGAMS_DISCARD_CMD,
[["path", trgFile], ["execute", "0"]])
refStatFile = "ref/ngamsDiscardCmdTest_test_NormalExec_2_1_ref"
self.checkFilesEq(refStatFile, tmpStatFile, illStatDoc % "1")
tmpStatFile = sendExtCmd(8888, NGAMS_DISCARD_CMD,
[["path", trgFile], ["execute", "1"]])
refStatFile = "ref/ngamsDiscardCmdTest_test_NormalExec_2_2_ref"
self.checkFilesEq(refStatFile, tmpStatFile, illStatDoc % "2")
def test_ArchiveCmd_Err_5_1(self):
"""
Synopsis:
Issue ARCHIVE Command/socket breaks while writing data on it.
Description:
The purpose of the test is to verify that the C-API handles
properly the situation where the socket connection breaks while
data is being written on it (during an Archive Push Request).
Expected Result:
The C-API should detect that the write socket connection breaks, and
should produce the appropriate error message.
Test Steps:
- Start special instance of server which terminates itself while
the client writes the data to the server.
- Issue Archive Push Request with a big file to the server.
- Verify that the proper error response is produced by the C-API.
Remarks:
...
"""
rmFile("tmp/reqCallBack_tmp")
saveInFile("tmp/reqCallBack_tmp", "reqCallBack_SrvCrash1")
self.prepExtSrv(srvModule="ngamsSrvTestDynReqCallBack")
cpFile("src/WFI-TEST.fits.Z", "tmp/WFI-TEST_tmp.fits.Z")
rmFile("tmp/WFI-TEST_tmp.fits")
subprocess.check_call(['uncompress', 'tmp/WFI-TEST_tmp.fits.Z'])
out = _execCClient(unpackXmlStat=0,
pars=[["-port", "8888"], ["-cmd", "ARCHIVE"],
["-fileUri", "tmp/WFI-TEST_tmp.fits"]])
tmpStatFile = saveInFile(None, filterOutLines(out, ["Host"]))
refStatFile = "ref/ngamsCClientTest_test_ArchiveCmd_Err_5_1_ref"
self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect handling " +\
"of broken write socket in C-Client/API")
def queue_smallfits(self, basename='test.fits'):
cpFile(self.resource('src/SmallFile.fits'),
os.path.join(self.monitor.queue_dir, basename))
def archiveFromFile(srvObj,
filename,
noReplication=0,
mimeType=None,
reqPropsObj=None):
"""
Archive a file directly from a file as source.
srvObj: Reference to NG/AMS Server Object (ngamsServer).
filename: Name of file to archive (string).
noReplication: Flag to enable/disable replication (integer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
Returns: Execution result object of DAPI
"""
T = TRACE()
logger.debug("Archiving file: %s", filename)
logger.debug("Mimetype used is %s", mimeType)
if (reqPropsObj):
logger.debug("Request Properties Object given - using this")
reqPropsObjLoc = reqPropsObj
else:
logger.debug("No Request Properties Object given - creating one")
reqPropsObjLoc = ngamsArchiveUtils.ngamsReqProps.ngamsReqProps()
stagingFile = filename
try:
if (mimeType == None):
mimeType = ngamsHighLevelLib.determineMimeType(
srvObj.getCfg(), filename)
archive_start = time.time()
# Prepare dummy ngamsReqProps object (if an object was not given).
if (not reqPropsObj):
reqPropsObjLoc.setMimeType(mimeType)
reqPropsObjLoc.setStagingFilename(filename)
reqPropsObjLoc.setHttpMethod(NGAMS_HTTP_GET)
reqPropsObjLoc.setCmd(NGAMS_ARCHIVE_CMD)
reqPropsObjLoc.setSize(os.path.getsize(filename))
reqPropsObjLoc.setFileUri(NGAMS_HTTP_FILE_URL + filename)
reqPropsObjLoc.setNoReplication(noReplication)
# If no target disk is defined, find one suitable disk.
if (not reqPropsObjLoc.getTargDiskInfo()):
try:
trgDiskInfo = ngamsArchiveUtils.ngamsDiskUtils.\
findTargetDisk(srvObj.getHostId(),
srvObj.getDb(), srvObj.getCfg(),
mimeType, 0,
reqSpace=reqPropsObjLoc.getSize())
reqPropsObjLoc.setTargDiskInfo(trgDiskInfo)
# copy the file to the staging area of the target disk
stagingFile = trgDiskInfo.getMountPoint(
) + '/staging/' + os.path.basename(filename)
cpFile(filename, stagingFile)
reqPropsObjLoc.setStagingFilename(stagingFile)
except Exception, e:
errMsg = str(e) + ". Attempting to archive local file: " +\
filename
ngamsPlugInApi.notify(srvObj, NGAMS_NOTIF_NO_DISKS,
"NO DISKS AVAILABLE", errMsg)
raise Exception, errMsg
# Set the log cache to 1 during the handling of the file.
plugIn = srvObj.getMimeTypeDic()[mimeType]
logger.debug("Invoking DAPI: %s to handle file: %s", plugIn,
stagingFile)
plugInMethod = loadPlugInEntryPoint(plugIn)
resMain = plugInMethod(srvObj, reqPropsObjLoc)
# Move the file to final destination.
st = time.time()
mvFile(reqPropsObjLoc.getStagingFilename(),
resMain.getCompleteFilename())
iorate = reqPropsObjLoc.getSize() / (time.time() - st)
ngamsArchiveUtils.postFileRecepHandling(srvObj, reqPropsObjLoc,
resMain, trgDiskInfo)
def archiveFromFile(srvObj,
filename,
noReplication=0,
mimeType=None,
reqPropsObj=None):
"""
Archive a file directly from a file as source.
srvObj: Reference to NG/AMS Server Object (ngamsServer).
filename: Name of file to archive (string).
noReplication: Flag to enable/disable replication (integer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
Returns: Execution result object of DAPI
"""
T = TRACE()
logger.debug("Archiving file: %s", filename)
logger.debug("Mimetype used is %s", mimeType)
if (reqPropsObj):
logger.debug("Request Properties Object given - using this")
reqPropsObjLoc = reqPropsObj
else:
logger.debug("No Request Properties Object given - creating one")
reqPropsObjLoc = ngamsArchiveUtils.ngamsReqProps.ngamsReqProps()
stagingFile = filename
try:
if (mimeType == None):
mimeType = ngamsHighLevelLib.determineMimeType(
srvObj.getCfg(), filename)
archive_start = time.time()
# Prepare dummy ngamsReqProps object (if an object was not given).
if (not reqPropsObj):
reqPropsObjLoc.setMimeType(mimeType)
reqPropsObjLoc.setStagingFilename(filename)
reqPropsObjLoc.setHttpMethod(NGAMS_HTTP_GET)
reqPropsObjLoc.setCmd(NGAMS_ARCHIVE_CMD)
reqPropsObjLoc.setSize(os.path.getsize(filename))
reqPropsObjLoc.setFileUri(NGAMS_HTTP_FILE_URL + filename)
reqPropsObjLoc.setNoReplication(noReplication)
# If no target disk is defined, find one suitable disk.
if (not reqPropsObjLoc.getTargDiskInfo()):
try:
trgDiskInfo = ngamsArchiveUtils.ngamsDiskUtils.\
findTargetDisk(srvObj.getHostId(),
srvObj.getDb(), srvObj.getCfg(),
mimeType, 0,
reqSpace=reqPropsObjLoc.getSize())
reqPropsObjLoc.setTargDiskInfo(trgDiskInfo)
# copy the file to the staging area of the target disk
stagingFile = trgDiskInfo.getMountPoint(
) + '/staging/' + os.path.basename(filename)
cpFile(filename, stagingFile)
reqPropsObjLoc.setStagingFilename(stagingFile)
except Exception as e:
errMsg = str(e) + ". Attempting to archive local file: " +\
filename
ngamsPlugInApi.notify(srvObj, NGAMS_NOTIF_NO_DISKS,
"NO DISKS AVAILABLE", errMsg)
raise Exception(errMsg)
# Set the log cache to 1 during the handling of the file.
plugIn = srvObj.getMimeTypeDic()[mimeType]
logger.debug("Invoking DAPI: %s to handle file: %s", plugIn,
stagingFile)
plugInMethod = loadPlugInEntryPoint(plugIn)
resMain = plugInMethod(srvObj, reqPropsObjLoc)
# Move the file to final destination.
st = time.time()
mvFile(reqPropsObjLoc.getStagingFilename(),
resMain.getCompleteFilename())
iorate = reqPropsObjLoc.getSize() / (time.time() - st)
ngamsArchiveUtils.postFileRecepHandling(srvObj, reqPropsObjLoc,
resMain, trgDiskInfo)
except Exception as e:
# If another error occurs, than one qualifying for Back-Log
# Buffering the file, we have to log an error.
if (ngamsHighLevelLib.performBackLogBuffering(srvObj.getCfg(),
reqPropsObjLoc, e)):
logger.exception(
"Tried to archive local file %s, keeping original file",
filename)
return [NGAMS_FAILURE, str(e), NGAMS_FAILURE]
else:
logger.exception("Tried to archive local file %s, " + \
"moving it to Bad Files Directory " + \
"-- cannot be handled", filename)
ngamsHighLevelLib.moveFile2BadDir(srvObj.getCfg(), filename)
# Remove pickle file if available.
pickleObjFile = filename + "." + NGAMS_PICKLE_FILE_EXT
if (os.path.exists(pickleObjFile)):
logger.debug("Removing Back-Log Buffer Pickle File: %s",
pickleObjFile)
rmFile(pickleObjFile)
return [NGAMS_FAILURE, str(e), NGAMS_FAILURE]
# If the file was handled successfully, we remove it from the
# Back-Log Buffer Directory unless the local file was a log-file
# in which case we leave the cleanup to the Janitor-Thread.
if stagingFile.find('LOG-ROTATE') > -1:
logger.debug("Successfully archived local file: %s", filename)
else:
logger.debug(
"Successfully archived local file: %s. Removing staging file.",
filename)
rmFile(stagingFile)
rmFile(stagingFile + "." + NGAMS_PICKLE_FILE_EXT)
logger.debug("Archived local file: %s. Time (s): %.3f", filename,
time.time() - archive_start)
return (resMain, trgDiskInfo, iorate)