def test_ArchiveCmd_Err_1(self):
"""
Synopsis:
Issue ARCHIVE Command/request times out.
Description:
The purpose of the test is to check that a request that times out
is handled properly by the C-Client/API and a proper error message
is produced.
Expected Result:
After the given timeout, the C-Client/API should generate a timeout
error message.
Test Steps:
- Start speciel instance of server where Archive Requests blocks.
- Issue Archive Request (small file) specifying a timeout of 10s.
- Capture the output from the ngamsCClient, filter this, and
check that the proper error message has been generated.
Remarks:
...
"""
rmFile("tmp/reqCallBack_tmp")
saveInFile("tmp/reqCallBack_tmp", "reqCallBack_BlockCmds1")
self.prepExtSrv(srvModule="ngamsSrvTestDynReqCallBack")
out = _execCClient(unpackXmlStat=0,
pars=[["-port", "8888"], ["-cmd", "ARCHIVE"],
["-fileUri", "src/SmallFile.fits"],
["-timeOut", "5"]])
tmpStatFile = saveInFile(None, filterOutLines(out, ["Host"]))
refStatFile = "ref/ngamsCClientTest_test_ArchiveCmd_Err_1_ref"
self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect handling " +\
"of timeout of Archive Request in C-Client/API")
def test_ArchiveCmd_Err_4_1(self):
"""
Synopsis:
Correct HTTP response, but illegal NG/AMS XML status document.
Description:
The purpose of the test is to verify that the C-Client/API handles
correctly the situation where an incorrectly formatted XML status
response is contained in the response from the server and the
proper error code generated by the C-API.
Expected Result:
The C-API detects the wrongly formatted NG/AMS XML status document,
and produces the appropriate error code.
Test Steps:
- Start special instance of the server which generates an HTTP
response with a corrupt XML NG/AMS status document.
- Issue an Archive Request via the ngamsCClient.
- Compare the output from ngamsCClient to check that the invalid
response was correctly handled.
Remarks:
...
"""
# TODO: From V4.0, this test case produces the error:
#
# Error Code: -103
# Message: Invalid reply from data server
#
# - and not:
#
# Error Code: -4
# Message: Problem communicating with server
# Status: FAILURE
#
# - as for previous version.
#
# This should be investigated and resolved.
rmFile("tmp/reqCallBack_tmp")
saveInFile("tmp/reqCallBack_tmp", "reqCallBack_IllegalResp")
self.prepExtSrv(srvModule="ngamsSrvTestDynReqCallBack")
httpResp = "HTTP/1.0 200 OK\015\012" +\
"Server: NGAMS/v2.3/2004-07-12T11:39:39\015\012" +\
"Date: Thu, 7 Oct 2004 16:20:28 GMT\015\012" +\
"Expires: Thu, 7 Oct 2004 16:20:28 GMT\015\012" +\
"Content-Type: text/xml\015\012" +\
"Content-Length: 36\015\012" +\
"\015\012" +\
"COMPLETELY CORRUPT NG/AMS XML STATUS"
saveInFile("tmp/ngamsServerTestIllegalResp_tmp", httpResp)
out = _execCClient(unpackXmlStat=0,
pars=[["-port", "8888"], ["-cmd", "ARCHIVE"],
["-fileUri", "src/SmallFile.fits"]])
tmpStatFile = saveInFile(None, filterOutLines(out, ["Host"]))
refStatFile = "ref/ngamsCClientTest_test_ArchiveCmd_Err_4_1_ref"
self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect handling " +\
"of corrupt server HTTP response in C-Client/API")
def remDisk(srvObj, reqPropsObj, diskId, execute):
"""
Select a disk for removal and remove the information if so specified
from the NGAS DB and delete all information on the disk.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
diskId: ID of disk. Complete ID must be specified. I.e., no
wildcards are handled (string).
execute: If set to 1 the information about the disk will be deleted.
Otherwise only the information about the disk selected for
deletion will be queried (integer/0|1).
Returns: Status object contained information about disk
selected for deletion/deleted (ngamsStatus).
"""
tmpFilePat = ngamsHighLevelLib.genTmpFilename(srvObj.getCfg(),
"REMDISK_CMD")
try:
status = _remDisk(srvObj, reqPropsObj, diskId, execute, tmpFilePat)
return status
finally:
rmFile(tmpFilePat + "*")
def checkTarball(filename):
"""
Check that the tarball is correct
filename: Name of tarball file (string).
Returns: Void.
"""
err = 0
tmpFilename = filename.replace(":", "_") + "_tmp"
try:
execCmd("ln -s " + filename + " " + tmpFilename)
stat, out, _ = execCmd("tar -tf " + tmpFilename)
rmFile(tmpFilename)
if (stat != 0):
errMsg = str(out).replace("\n", " --- ")
err = 1
except Exception as e:
rmFile(tmpFilename)
errMsg = str(e)
err = 1
if (err):
errMsg = "Error checking tarball: " + errMsg
errMsg = genLog("NGAMS_ER_DAPI_BAD_FILE",
[filename, "ngasTarBallPlugIn", errMsg])
raise Exception(errMsg)
def retrieveFiles(diskIdList):
"""
Retrieve all files associated to a list of Disk IDs.
diskIdList: List of Disk IDs (list).
Returns: Void.
"""
dbCon = ngamsDb.ngamsDb("ESOECF", "ngas", "ngas", "bmdhc19wdw==")
client = ngamsPClient.ngamsPClient("jewel1", 7777)
for diskId in diskIdList:
dbCur = dbCon.getFileInfoList("IBM-DTLA-307075-YSDYSG3X871")
while (1):
fileList = dbCur.fetch(100)
if (not fileList): break
for fileInfo in fileList:
fileId = fileInfo[ngamsDb.NGAS_FILES_FILE_ID]
fileVersion = fileInfo[ngamsDb.NGAS_FILES_FILE_VER]
print "Retrieving file with File ID/Version: %s/%d" %\
(fileId, fileVersion)
startTime = time.time()
res = client.retrieve(fileId, fileVersion,"/tmp/TestFile")
deltaTime = (time.time() - startTime)
if (res.getStatus() == NGAMS_SUCCESS):
format = " - Successfully retrieved file with " +\
"File ID/Version: %s/%d/Time: %.3fs"
print format % (fileId, fileVersion, deltaTime)
else:
format = " - ERROR retrieving file with " +\
"File ID/Version: %s/%d: %s/Time: %.3fs"
print format % (fileId, fileVersion,
str(res.getMessage()).replace("\n", " - "),
deltaTime)
rmFile("/tmp/TestFile")
def _delFile(srvObj, path, hostId, execute):
"""
Delete (remove from disk) the file specified by the path on the given host.
srvObj: Reference to NG/AMS server class object (ngamsServer).
path: Path of file (string).
hostId: ID of host where file is stored (string).
Returns: Message with info about the execution (string).
"""
if (hostId and (hostId != srvObj.getHostId())):
raise Exception, "DISCARD Command can only be executed locally!"
# Handle the discard request locally.
ngasHostId = "%s:%d" % (getHostName(), srvObj.getCfg().getPortNo())
if (os.path.exists(path)):
if (not ngamsLib.fileRemovable(path)):
return genLog("NGAMS_ER_DISCARD_NOT_REM", [path, ngasHostId])
if (not execute):
msg = genLog("NGAMS_INFO_DISCARD_GRANTED", [path, ngasHostId])
else:
rmFile(path)
if (os.path.exists(path)):
return genLog("NGAMS_ER_DISCARD_FAILED",
[path, ngasHostId, "Not removable"])
else:
msg = genLog("NGAMS_INFO_DISCARD_OK", [path, ngasHostId])
else:
return genLog("NGAMS_ER_DISCARD_NOT_FOUND", [path, ngasHostId])
return msg
def _handleFileList(srvObj, reqPropsObj, httpRef):
"""
Handle STATUS?file_list... Command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: The File List ID allocated for this request (string).
"""
T = TRACE()
# Should a lower limit for the ingestion date be taken into account.
fromIngDate = None
if (reqPropsObj.hasHttpPar("from_ingestion_date")):
tmpTromIngDate = reqPropsObj.getHttpPar("from_ingestion_date")
fromIngDate = fromiso8601(tmpTromIngDate)
# Handle the unique flag. If this is specified, only information for unique
# File ID/Version pairs are returned.
unique = False
if (reqPropsObj.hasHttpPar("unique")):
unique = int(reqPropsObj.getHttpPar("unique"))
# Dump the file information needed.
fileListId = genUniqueId()
dbmBaseName = STATUS_FILE_LIST_DBM_TAG % fileListId
fileInfoDbmName = ngamsHighLevelLib.genTmpFilename(srvObj.getCfg(),
dbmBaseName)
# Dump the file info from the DB. Deal with uniqueness quickly
# (instead of using a new file like before)
try:
fileInfoDbm = ngamsDbm.ngamsDbm(fileInfoDbmName, 0, 1)
fileCount = 1
unique_files = set()
for f in srvObj.db.files_in_host(srvObj.getHostId(),
from_date=fromIngDate):
if unique:
key = str('%s_%d' % (f[2], f[3]))
if key in unique_files:
continue
else:
key = str(fileCount)
fileInfoDbm.add(key, f)
fileCount += 1
except Exception as e:
rmFile(fileInfoDbmName)
msg = "Problem generating file list for STATUS Command. " +\
"Parameters: from_ingestion_date=%s. Error: %s" %\
(str(fromIngDate), str(e))
raise Exception(msg)
return fileListId
def retrieve_container(port):
tgt_dir = tmp_path('tgt')
self.cretrieve(port,
container_name,
targetDir=tgt_dir,
as_tar=as_tar)
self._assertEqualsDir(src_root, os.path.join(tgt_dir, 'toplevel'))
rmFile(tgt_dir)
def test_ArchiveCmd_Err_3_1(self):
"""
Synopsis:
Handling of corrupted HTTP response.
Description:
The purpose of the test is test that a corrupted HTTP response is
properly handled by the C-Client/API.
The response is corrupted such that it only contains a '\015\012'
({Slash r}{Slash n}).
Expected Result:
The corrupt HTTP response cannot be unpacked/interpreted as an
XML document. This should be detected by the C-API and a proper
error message returned.
Test Steps:
- Start special instance of the server class, which produces
an illegal response.
- Issue an ARCHIVE Command via the ngamsCClient.
- Check that the output produced on stdout refers to the appropriate
error message.
Remarks:
...
"""
# TODO: From V4.0, this test case produces the error:
#
# Error Code: -4
# Message: Problem communicating with server
#
# - and not:
#
# Error Code: -103
# Message: Invalid reply from data server
#
# - as for previous version.
#
# This should be investigated and resolved.
rmFile("tmp/reqCallBack_tmp")
saveInFile("tmp/reqCallBack_tmp", "reqCallBack_IllegalResp")
rmFile("tmp/ngamsServerTestIllegalResp_tmp")
saveInFile("tmp/ngamsServerTestIllegalResp_tmp", "\015\012")
self.prepExtSrv(srvModule="ngamsSrvTestDynReqCallBack")
out = _execCClient(unpackXmlStat=0,
pars=[["-port", "8888"], ["-cmd", "ARCHIVE"],
["-fileUri", "src/SmallFile.fits"]])
tmpStatFile = saveInFile(None, filterOutLines(out, ["Host"]))
refStatFile = "ref/ngamsCClientTest_test_ArchiveCmd_Err_3_1_ref"
self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect handling " +\
"of corrupt server HTTP response in C-Client/API")
def checkCleanDirs(startDir,
dirExp,
fileExp,
useLastAccess):
"""
Check a tree of directories. Delete all empty directories older than
the given Directory Expiration. Also files are deleted if the file is
older than File Expiration given.
startDir: Starting directory. The function will move downwards from
this starting point (string).
dirExp: Expiration time in seconds for directories. Empty
directories older than this time are deleted (integer).
fileExp: Expiration time in seconds for file. Empty file older than
this time are deleted (integer).
useLastAccess: Rather than using the creation time as reference, the
last modification and access date should be used
(integer/0|1).
Returns: Void.
"""
timeNow = time.time()
# TODO: Potential memory bottleneck. Use 'find > file' as for REGISTER
# Command.
entryList = glob.glob(startDir + "/*")
# Work down through the directories in a recursive manner. If some
# directories are not deleted during this run because they have contents,
# they might be deleted during one of the following runs.
for entry in entryList:
stat = os.stat(entry)
if (not useLastAccess):
refTime = stat.st_ctime # creation time
else:
refTime1 = stat.st_mtime # modification time
refTime2 = stat.st_atime # access time
if (refTime1 > refTime2):
refTime = refTime1
else:
refTime = refTime2
if (os.path.isdir(entry)):
checkCleanDirs(entry, dirExp, fileExp, useLastAccess)
tmpGlobRes = glob.glob(entry + "/*")
if (tmpGlobRes == []):
if ((timeNow - refTime) > dirExp):
logger.debug("Deleting temporary directory: %s", entry)
rmFile(entry)
else:
if (fileExp):
if ((timeNow - refTime) > fileExp):
logger.debug("Deleting temporary file: %s", entry)
rmFile(entry)
def test_VolumeDir_01(self):
"""
Synopsis:
Grouping of data volumes under the Volume Dir in the NGAS Root Dir.
Description:
See ngamsArchiveCmdTest.test_VolumeDir_01().
This tests verifies that the files archived into this structure can
be received.
Expected Result:
When the server goes Online, it should accept the given directory
structure and it should be possible to archive files into this
structureand subsequently to receive them.
Test Steps:
- Create the volume dirs from an existing structure.
- Start server with configuration specifying the Volumes Dir in which
all volumes will be hosted.
- Archive a FITS file.
- Retrieve the files and check that they are OK.
Remarks:
...
"""
# Create basic structure.
ngasRootDir = tmp_path("NGAS")
rmFile(ngasRootDir)
checkCreatePath(ngasRootDir)
subprocess.check_call(
['tar', 'zxf',
self.resource('src/volumes_dir.tar.gz')])
mvFile('volumes', ngasRootDir)
# Create configuration, start server.
self.prepExtSrv(delDirs=0, cfgFile='src/ngamsCfg_VolumeDirectory.xml')
# Archive a file.
stat = self.archive("src/SmallFile.fits")
self.assert_status_ref_file(
"ref/ngamsRetrieveCmdTest_test_VolumeDir_01_01_ref", stat)
# Check that the target files have been archived in their
# appropriate locations.
trgFile = tmp_path("test_VolumeDir_01_tmp")
refFile = "src/SmallFile.fits"
outFilePath = tmp_path("SmallFile.fits")
stat = self.retrieve("TEST.2001-05-08T15:25:00.123",
targetFile=trgFile)
# unzip the the file and diff against original
unzip(trgFile, outFilePath)
self.checkFilesEq(refFile, outFilePath, "Retrieved file incorrect")
def main():
"""
Main function to execute the tool
"""
# Parse input parameters
access_code = ""
file_list_file = ""
dcc_message_file = ""
execute = 0
notification_email = None
index = 1
while index < len(sys.argv):
par = sys.argv[index].upper()
try:
if par == "-ACCESSCODE":
index += 1
access_code = sys.argv[index]
elif par == "-FILELIST":
index += 1
file_list_file = sys.argv[index]
elif par == "-DCCMSG":
index += 1
dcc_message_file = sys.argv[index]
elif par == "-EXECUTE":
execute = 1
elif par == "-NOTIFEMAIL":
index += 1
notification_email = sys.argv[index]
else:
sys.exit(1)
index += 1
except Exception as e:
print("\nProblem executing the File Discard Tool: {:s}\n".format(str(e)))
print(correct_usage())
sys.exit(1)
if notification_email is None:
notification_email = ngasUtilsLib.get_parameter_ngas_resource_file(ngasUtilsLib.NGAS_RC_PAR_NOTIF_EMAIL)
if dcc_message_file and not file_list_file:
file_list_file = os.path.join(tempfile.gettempdir(), "ngasDiscardFiles.tmp")
rmFile(file_list_file)
ngasUtilsLib.dcc_message_to_file_list(dcc_message_file, file_list_file)
try:
if not file_list_file:
print(correct_usage())
raise Exception("Incorrect command line parameter(s) given!")
if not access_code:
access_code = ngasUtilsLib.console_input("Enter Access Code:")
ngasUtilsLib.check_access_code(access_code)
discard_files(file_list_file, execute, notification_email)
except Exception as e:
print("\nProblem encountered:\n\n" + str(e) + " -- bailing out\n")
def compress(reqPropsObj, parDic):
"""
Compress the file if required.
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
parDic: Dictionary with parameters for the DAPI. This is generated
with ngamsPlugInApi.parseDapiPlugInPars() (Dictionary).
Returns: Tupe containing uncompressed filesize, archived filesize
and the format (mime-type) of the resulting data file
(tuple).
"""
stFn = reqPropsObj.getStagingFilename()
uncomprSize = ngamsPlugInApi.getFileSize(stFn)
mime = reqPropsObj.getMimeType()
compression = parDic.get("compression")
if _compress_data(parDic):
logger.debug("Compressing file: %s using: %s", stFn, compression)
# Compress *and* calculate checksum on compressed stream
# The value crc_name depends on the server configuration and whether the
# user requested a different variant (see ngamsArchiveUtils)
gzip_name = '%s.gz' % stFn
crc_info = None
if 'crc_name' in reqPropsObj:
crc_info = ngamsFileUtils.get_checksum_info(
reqPropsObj['crc_name'])
compress_start = time.time()
with open(stFn, 'rb') as f:
crc = ngamsLib.gzip_compress(f,
gzip_name,
65536,
crc_info=crc_info)
compress_time = time.time() - compress_start
reqPropsObj.setStagingFilename(gzip_name)
rmFile(stFn)
mime = 'application/x-gfits'
compression = 'gzip --no-name'
logger.debug("File compressed: %s Time: %.3fs", gzip_name,
compress_time)
else:
compression = ''
crc = None
archFileSize = ngamsPlugInApi.getFileSize(reqPropsObj.getStagingFilename())
return uncomprSize, archFileSize, mime, compression, crc
def monitorMemUsage(parDic):
"""
"""
rmFile(parDic["OUTFILE"])
fo = open(parDic["OUTFILE"], "w")
memUsageCmd = "/usr/bin/pmap %d" % parDic["PID"]
while (1):
stat, out = commands.getstatusoutput(memUsageCmd)
memUsage = out.split("\n")[-1].split(" ")[-1][0:-1]
logLine = "%s %s\n" % (PccUtTime.TimeStamp().getTimeStamp(), memUsage)
fo.write(logLine)
print logLine[:-1]
fo.flush()
time.sleep(parDic["INTERVAL"])
fo.close()
def _openDbSnapshot(ngamsCfgObj,
mtPt):
"""
Open a bsddb file DB. If the file exists and this is not
a read-only NGAS system the file is opened for reading and writing.
If this is a read-only NGAS system it is only opened for reading.
If the file DB does not exist, a new DB is created.
If the file DB does not exist and this is a read-only NGAS system,
None is returned.
The name of the DB file is:
<Disk Mount Point>/NGAMS_DB_DIR/NGAMS_DB_NGAS_FILES
ngamsCfgObj: NG/AMS Configuration Object (ngamsConfig).
mtPt: Mount point (string).
Returns: File DB object (bsddb|None).
"""
snapShotFile = os.path.normpath(mtPt + "/" + NGAMS_DB_DIR + "/" +\
NGAMS_DB_NGAS_FILES)
checkCreatePath(os.path.normpath(mtPt + "/" + NGAMS_DB_CH_CACHE))
if (os.path.exists(snapShotFile)):
if (_updateSnapshot(ngamsCfgObj)):
# Open the existing DB Snapshot for reading and writing.
snapshotDbm = bsddb.hashopen(snapShotFile, "w")
else:
# Open only for reading.
snapshotDbm = bsddb.hashopen(snapShotFile, "r")
else:
if (_updateSnapshot(ngamsCfgObj)):
# Create a new DB Snapshot.
snapshotDbm = bsddb.hashopen(snapShotFile, "c")
else:
# There is no DB Snapshot and it is not possible to
# create one - the check cannot be carried out.
snapshotDbm = None
# Remove possible, old /<mt pt>/.db/NgasFiles.xml snapshots.
# TODO: Remove when it can be assumed that all old XML snapshots have
# been removed.
rmFile(os.path.normpath(mtPt + "/" + NGAMS_DB_DIR + "/NgasFiles.xml"))
return snapshotDbm
def _cloneThread(srvObj,
cloneListDbmName,
tmpFilePat,
targetDiskId="",
reqPropsObj=None,
dummyPar=None):
"""
Function that carried out the actual cloning process of the files
referenced to in the 'cloneList'
srvObj: Reference to instance of Server Object (ngamsServer).
cloneListDbmName: Name of DBM containing the information about
the files to be cloned. This DB has an index number
as key pointing to pickled information about each file.
This pickled information is
[<File Info Object>, <Host ID>, <Mount Point>]
(string)
tmpFilePat: File pattern to be used for generating temporary
files (string).
targetDiskId: ID of disk to where the files cloned should be written
(string).
reqPropsObj: If an NG/AMS Request Properties Object is given, the
Request Status will be updated as the request is carried
out (ngamsReqProps).
Returns: Void.
"""
logger.info("Cloning Thread carrying out Clone Request ...")
try:
_cloneExec(srvObj, cloneListDbmName, tmpFilePat, targetDiskId,
reqPropsObj)
rmFile(tmpFilePat + "*")
logger.info("Processing of Clone Request completed")
thread.exit()
except Exception:
rmFile(tmpFilePat + "*")
raise
def run(srvObj, stopEvt):
cfg = srvObj.getCfg()
logdir = os.path.dirname(cfg.getLocalLogFile())
# The LOG-ROTATE-<timestamp>.nglog.unsaved pattern is produced by the
# ngamsServer.NgasRotatingFileHandler class, which is the file handler
# attached to the root logger in the server process
logger.debug("Checking if there are unsaved rotated logfiles")
for unsaved in sorted(
glob.glob(os.path.join(logdir, 'LOG-ROTATE-*.nglog.unsaved'))):
# Remove the .unsaved bit, leave the rest
fname = '.'.join(unsaved.split('.')[:-1])
try:
mvFile(unsaved, fname)
try_archiving(cfg, srvObj, fname)
except Exception as e:
mvFile(fname, unsaved)
if isinstance(e, socket.error) and e.errno == errno.ECONNREFUSED:
# this is expected when the server is just starting up,
# let's ignore for now
logger.warning(
"Server not up yet, postponing processing of %s", unsaved)
continue
raise
# Do additional things with our logfiles
for plugin in get_logfile_handler_plugins(cfg):
try:
plugin(srvObj, fname)
except:
logger.exception("Error while handling logfile %s", fname)
logger.debug("Check if there are old rotated logfiles to remove ...")
max_rotations = max(min(cfg.getLogRotateCache(), 100), 0)
logfiles = glob.glob(os.path.join(logdir, 'LOG-ROTATE-*.nglog'))
logfiles.sort()
for f in logfiles[max_rotations:]:
logger.info("Removing rotated logfile %s", f)
rmFile(f)
def checkTarball(filename):
"""
Check that the tarball is correct
filename: Name of tarball file (string).
Returns: Void.
"""
err = 0
tmpFilename = filename.replace(":", "_") + "_tmp"
try:
commands.getstatusoutput("ln -s " + filename + " " + tmpFilename)
stat, out = commands.getstatusoutput("tar -tf " + tmpFilename)
rmFile(tmpFilename)
if (stat != 0):
errMsg = str(out).replace("\n", " --- ")
err = 1
except Exception, e:
rmFile(tmpFilename)
errMsg = str(e)
err = 1
def _registerThread(srvObj,
fileListDbmName,
tmpFilePat,
diskInfoDic,
reqPropsObj=None,
dummyPar=None):
"""
See documentation for _registerExec().
The purpose of this function is merely to excapsulate the actual
cloning to make it possible to clean up in case an exception is thrown.
"""
try:
_registerExec(srvObj, fileListDbmName, tmpFilePat, diskInfoDic,
reqPropsObj)
rmFile(tmpFilePat + "*")
return
except Exception:
logger.exception("Exception raised in Register Sub-Thread")
rmFile(tmpFilePat + "*")
raise
def ngamsTileCompress(filename):
"""
Tile compress the referenced file.
filename: Filename (string).
Returns: Void.
"""
tmpFilename = filename + ".tmp"
try:
comprCmd = "imcopy %s '%s[compress]'" % (filename, tmpFilename)
logger.debug("Command to tile compress file: %s", comprCmd)
stat, out = commands.getstatusoutput(comprCmd)
if (stat != 0):
msg = "Error compressing file: %s. Error: %s" %\
(filename, stat.replace("\n", " "))
raise Exception, msg
mvFile(tmpFilename, filename)
logger.debug("Successfully tile compressed file: %s", filename)
except Exception, e:
rmFile(tmpFilename)
raise Exception, e
def test_RetrieveCmd_Err_1_2(self):
"""
Synopsis:
Issue RETRIEVE Command/server dies during initial handling.
Description:
Check that the situation where the server dies during the initial
handling of a Retrieve Request is correctly handled by the C-API.
Expected Result:
The C-API should detect that the server died (=broken socket
connection) and should produce the appropriate error code, which is
printed on stdout by the ngamsCClient.
Test Steps:
- Start a special instance of the server, which kills itself
when receiving a RETRIEVE Command.
- Archive a file.
- Issue a RETRIEVE Command to retrieve the archived file.
- Verify that the proper output is produced by ngamsCClient indicating
the problem.
Remarks:
...
"""
rmFile("tmp/reqCallBack_tmp")
saveInFile("tmp/reqCallBack_tmp", "reqCallBack_SrvCrash2")
self.prepExtSrv(srvModule="ngamsSrvTestDynReqCallBack")
out =\
_execCClient(unpackXmlStat = 0,
pars = [["-port", "8888"],
["-cmd", "RETRIEVE"],
["-fileId", "TEST.2001-05-08T15:25:00.123"],
["-timeOut", "10"]])
tmpStatFile = saveInFile(None, filterOutLines(out, ["Host"]))
refStatFile = "ref/ngamsCClientTest_test_RetrieveCmd_Err_1_2_ref"
self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect handling " +\
"of timeout of Retrieve Request in C-Client/API")
def test_RetrieveCmd_Err_1_1(self):
"""
Synopsis:
Issue RETRIEVE Command/request times out.
Description:
The purpose of the test is to verify that the situation where a
Retrieve Request times out is correctly handled by the C-API.
Expected Result:
After the specified timeout is reached, the appropriate error code
is returned to the client.
Test Steps:
- Start special instance of the server, which blocks on a Retrieve
Request (no response send).
- Issue a Retrieve Request via the ngamsCClient with timeout 10s.
- Verify that after 10s the correct error code is returned by the
C-API (printed by ngamsCClient on stdout).
Remarks:
...
"""
rmFile("tmp/reqCallBack_tmp")
saveInFile("tmp/reqCallBack_tmp", "reqCallBack_AccArchiveBlock2")
self.prepExtSrv(srvModule="ngamsSrvTestDynReqCallBack")
sendPclCmd(port=8888).archive("src/SmallFile.fits")
out =\
_execCClient(unpackXmlStat = 0,
pars = [["-port", "8888"],
["-cmd", "RETRIEVE"],
["-fileId", "TEST.2001-05-08T15:25:00.123"],
["-timeOut", "10"]])
tmpStatFile = saveInFile(None, filterOutLines(out, ["Host"]))
refStatFile = "ref/ngamsCClientTest_test_RetrieveCmd_Err_1_1_ref"
self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect handling " +\
"of timeout of Retrieve Request in C-Client/API")
def run(srvObj, stopEvt):
cfg = srvObj.getCfg()
logdir = os.path.dirname(cfg.getLocalLogFile())
# The LOG-ROTATE-<timestamp>.nglog.unsaved pattern is produced by the
# ngamsServer.NgasRotatingFileHandler class, which is the file handler
# attached to the root logger in the server process
logger.debug("Checking if there are unsaved rotated logfiles")
for unsaved in glob.glob(os.path.join(logdir,
'LOG-ROTATE-*.nglog.unsaved')):
# Remove the .unsaved bit, leave the rest
fname = '.'.join(unsaved.split('.')[:-1])
mvFile(unsaved, fname)
# Connect to the server and send a pull ARCHIVE request
if cfg.getArchiveRotatedLogfiles():
file_uri = "file://" + fname
host, port = srvObj.get_self_endpoint()
ngamsPClient.ngamsPClient(host,
port).archive(file_uri, 'ngas/nglog')
# Do additional things with our logfiles
for plugin in get_logfile_handler_plugins(cfg):
try:
plugin(srvObj, fname)
except:
logger.exception("Error while handling logfile %s", fname)
logger.debug("Check if there are old rotated logfiles to remove ...")
max_rotations = max(min(cfg.getLogRotateCache(), 100), 0)
logfiles = glob.glob(os.path.join(logdir, 'LOG-ROTATE-*.nglog'))
logfiles.sort()
for f in logfiles[max_rotations:]:
logger.info("Removing rotated logfile %s", f)
rmFile(f)
def test_ArchiveCmd_Err_2(self):
"""
Synopsis:
Issue ARCHIVE Command/server crashes (broken socket connection).
Description:
The purpose of the test is to verify the correct handling/behavior
of the C-Client/API in the case the socket connection to the server
breaks.
Expected Result:
During the request handling the socket connection is provoked to break
and the C-Client/API should detect this and shoudl produce the proper
error message.
Test Steps:
- Start special instance of the server which makes itself crash when
a request is received (to simulate a broken socket connection).
- Submit an ARCHIVE Command via the C-Client/API (small file).
- Check the result produced by the ngamsCClient on stdout that this
is as expected.
Remarks:
...
"""
rmFile("tmp/reqCallBack_tmp")
saveInFile("tmp/reqCallBack_tmp", "reqCallBack_SrvCrash1")
self.prepExtSrv(srvModule="ngamsSrvTestDynReqCallBack")
out = _execCClient(unpackXmlStat=0,
pars=[["-port", "8888"], ["-cmd", "ARCHIVE"],
["-fileUri", "src/SmallFile.fits"]])
tmpStatFile = saveInFile(None, filterOutLines(out, ["Host"]))
refStatFile = "ref/ngamsCClientTest_test_ArchiveCmd_Err_2_ref"
self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect handling " +\
"of crash of server in C-Client/API")
def test_ArchiveCmd_Err_3_3(self):
"""
Synopsis:
Issue ARCHIVE Command/server sends back a nonsense HTTP response.
Description:
The purpose of the test is test that a corrupted HTTP response is
properly handled by the C-Client/API.
The response is corrupted such that it contains 'noise'.
Expected Result:
The C-API should detect the illegally formatted response and return
the correct error code.
Test Steps:
- Start special instance of the server that generates a non-sense
HTTP response.
- Issue an Archive Request via the ngamsCClient.
- Check that the proper error message is produced by the ngamsCClient
Remarks:
...
"""
rmFile("tmp/reqCallBack_tmp")
saveInFile("tmp/reqCallBack_tmp", "reqCallBack_IllegalResp")
rmFile("tmp/ngamsServerTestIllegalResp_tmp")
saveInFile("tmp/ngamsServerTestIllegalResp_tmp", "f-423hcqfe-0")
self.prepExtSrv(srvModule="ngamsSrvTestDynReqCallBack")
out = _execCClient(unpackXmlStat=0,
pars=[["-port", "8888"], ["-cmd", "ARCHIVE"],
["-fileUri", "src/SmallFile.fits"]])
tmpStatFile = saveInFile(None, filterOutLines(out, ["Host"]))
refStatFile = "ref/ngamsCClientTest_test_ArchiveCmd_Err_3_3_ref"
self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect handling " +\
"of corrupt server HTTP response in C-Client/API")
def test_ArchiveCmd_Err_5_1(self):
"""
Synopsis:
Issue ARCHIVE Command/socket breaks while writing data on it.
Description:
The purpose of the test is to verify that the C-API handles
properly the situation where the socket connection breaks while
data is being written on it (during an Archive Push Request).
Expected Result:
The C-API should detect that the write socket connection breaks, and
should produce the appropriate error message.
Test Steps:
- Start special instance of server which terminates itself while
the client writes the data to the server.
- Issue Archive Push Request with a big file to the server.
- Verify that the proper error response is produced by the C-API.
Remarks:
...
"""
rmFile("tmp/reqCallBack_tmp")
saveInFile("tmp/reqCallBack_tmp", "reqCallBack_SrvCrash1")
self.prepExtSrv(srvModule="ngamsSrvTestDynReqCallBack")
cpFile("src/WFI-TEST.fits.Z", "tmp/WFI-TEST_tmp.fits.Z")
rmFile("tmp/WFI-TEST_tmp.fits")
subprocess.check_call(['uncompress', 'tmp/WFI-TEST_tmp.fits.Z'])
out = _execCClient(unpackXmlStat=0,
pars=[["-port", "8888"], ["-cmd", "ARCHIVE"],
["-fileUri", "tmp/WFI-TEST_tmp.fits"]])
tmpStatFile = saveInFile(None, filterOutLines(out, ["Host"]))
refStatFile = "ref/ngamsCClientTest_test_ArchiveCmd_Err_5_1_ref"
self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect handling " +\
"of broken write socket in C-Client/API")
def handleCmd(srvObj, reqPropsObj, httpRef):
"""
Handle Command QUERY to query the DB system used.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
T = TRACE()
# Get command parameters.
if not 'query' in reqPropsObj:
raise Exception("No query specified. Valid queries are: %s" %
(queries.keys(), ))
query = reqPropsObj.getHttpPar("query").lower()
if query not in queries.keys():
raise Exception("Invalid query specified. Valid queries are: %s" %
(queries.keys(), ))
out_format = None
if 'format' in reqPropsObj:
out_format = reqPropsObj["format"]
cursorId = None
if (reqPropsObj.hasHttpPar("cursor_id")):
cursorId = reqPropsObj.getHttpPar("cursor_id")
fetch = None
if (reqPropsObj.hasHttpPar("fetch")):
fetch = int(reqPropsObj.getHttpPar("fetch"))
# Select the SQL statement + pars to execute
colnames, sql = queries[query]
args = ()
if query in ('subscribers_like', 'files_like', 'files_location',
'lastver_location'):
param = '%'
if (reqPropsObj.hasHttpPar("like")):
param = reqPropsObj.getHttpPar("like")
args = (param, )
elif query == 'files_between':
param1 = param2 = ''
if 'start' in reqPropsObj:
param1 = reqPropsObj["start"]
if 'end' in reqPropsObj:
param2 = reqPropsObj["end"]
if param1 and param2:
args = (param1, param2)
elif param1:
sql = 'select * from ngas_files where ingestion_date >= {0}'
args = (param1, )
else:
sql = queries['files_list']
# Execute the query.
if not cursorId:
# TODO: Make possible to return an XML document
# TODO: Potential problem with very large result sets.
# Implement streaming result directly3
# rtobar, 2018 Feb: for the streaming result functionality to work
# correctly we need to support sending content with
# chunked transfer encoding first. This is not
# difficult to add on the server side, but needs
# to be also added on the client side (which I suppose
# means all languages, not only python, if we really
# need to maintain those).
# We already support proper cursors at the database
# access level, so it's not difficult to change
# the query below to use a cursor and work as a
# generator instead of returning the full list of
# results in one go.
res = srvObj.getDb().query2(sql, args=args)
if out_format in ("list", 'text'):
finalRes = formatAsList(res, colnames)
mimeType = NGAMS_TEXT_MT
elif out_format == "pickle":
finalRes = cPickle.dumps([res])
mimeType = NGAMS_PYTHON_PICKLE_MT
elif out_format == "json":
results = [{colname: val
for colname, val in zip(colnames, row)} for row in res]
finalRes = six.b(json.dumps(results, default=encode_decimal))
mimeType = NGAMS_JSON_MT
else:
finalRes = six.b(str(list(res)))
mimeType = NGAMS_PYTHON_LIST_MT
# Return the data and good bye.
httpRef.send_data(finalRes, mimeType)
return
# TODO:
#
#
#
# The rest seems to be very old functionality, which we probably
# want to drop at some point. I'm still keeping it here for the time being
# to be a good citizen and not break any (very potential) user.
#
#
#
elif (fetch):
cursorDbmFilename = genCursorDbmName(srvObj.getCfg().\
getRootDirectory(), cursorId)
if (not os.path.exists(cursorDbmFilename)):
logger.error("Illegal Cursor ID: %s or cursor expired", cursorId)
return []
try:
cursorDbm = ngamsDbm.ngamsDbm(cursorDbmFilename, writePerm=1)
count = 0
resSet = []
cursorIdx = cursorDbm.get(CURSOR_IDX)
while (count < fetch):
cursorIdx += 1
if (cursorDbm.hasKey(str(cursorIdx))):
res = cursorDbm.get(str(cursorIdx))
resSet.append(res)
else:
# -- no more results to return.
break
count += 1
cursorDbm.add(CURSOR_IDX, cursorIdx)
cursorDbm.sync()
del cursorDbm
# If all entries have been fetched, we delete the cursor DBM.
if (count < fetch):
rmFile(cursorDbmFilename + "*")
# Return the data.
# TODO: Make it possible to return ASCII List + XML.
httpRef.send_data(str(resSet), NGAMS_PYTHON_LIST_MT)
except Exception as e:
msg = "Error fetching from cursor with ID: %s. Error: %s"
raise Exception(msg % (cursorId, str(e)))
elif (query and cursorId):
logger.debug("Creating new cursor with ID: %s, query: %s", cursorId,
query)
cursorDbmFilename = genCursorDbmName(srvObj.getCfg().\
getRootDirectory(), cursorId)
cursorDbm = ngamsDbm.ngamsDbm(cursorDbmFilename, writePerm=1)
# Make the query in a cursor and dump the results into the DBM.
curObj = srvObj.getDb().dbCursor(query, args=args)
with curObj:
for res in curObj.fetch(1000):
cursorDbm.addIncKey(res)
cursorDbm.add(CURSOR_IDX, 0)
cursorDbm.sync()
del cursorDbm
# TODO: In this case no reply is generated??
else:
msg = "Error illegal combination of parameters. Correct syntax is: " +\
"QUERY?query=<Query>[&out_format=<Format (list)>] or " +\
"QUERY?query=<Query>&cursor_id=<ID> followed by N calls to " +\
"QUERY?cursor_id=<ID>&fetch=<Number of Elements>"
raise Exception(msg)
def handleCmd(srvObj,
reqPropsObj,
httpRef):
"""
Handle STATUS command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler object
(ngamsHttpRequestHandler).
Returns: Void.
"""
status = ngamsStatus.ngamsStatus()
status.setDate(toiso8601()).\
setVersion(getNgamsVersion()).setHostId(srvObj.getHostId()).\
setStatus(NGAMS_SUCCESS).\
setMessage("Successfully handled command STATUS").\
setState(srvObj.getState()).setSubState(srvObj.getSubState())
reqPropsObjRef = reqPropsObj
# Get the information requested.
diskId = ""
fileId = ""
fileVersion = ""
configurationFile = ""
fileAccess = ""
hostId = ""
requestId = ""
dbTime = ""
dbTimeReset = ""
fileList = ""
fileListId = ""
maxElements = 100000
if (reqPropsObj.hasHttpPar("disk_id")):
diskId = reqPropsObj.getHttpPar("disk_id")
if (reqPropsObj.hasHttpPar("file_id")):
fileId = reqPropsObj.getHttpPar("file_id")
if (reqPropsObj.hasHttpPar("file_version")):
fileVersion = int(reqPropsObj.getHttpPar("file_version"))
if (reqPropsObj.hasHttpPar("configuration_file")):
configurationFile = reqPropsObj.getHttpPar("configuration_file")
if (reqPropsObj.hasHttpPar("file_access")):
fileAccess = reqPropsObj.getHttpPar("file_access")
if (reqPropsObj.hasHttpPar("host_id")):
hostId = reqPropsObj.getHttpPar("host_id")
if (reqPropsObj.hasHttpPar("max_elements")):
maxElements = int(reqPropsObj.getHttpPar("max_elements"))
if (reqPropsObj.hasHttpPar("request_id")):
requestId = reqPropsObj.getHttpPar("request_id")
if (reqPropsObj.hasHttpPar("db_time")):
dbTime = True
if (reqPropsObj.hasHttpPar("db_time_reset")):
dbTimeReset = True
if (reqPropsObj.hasHttpPar("flush_log")):
# in the past this called flushLog()
# do we really want to keep that functionality? I doubt it
pass
if (reqPropsObj.hasHttpPar("file_list")):
fileList = int(reqPropsObj.getHttpPar("file_list"))
if (reqPropsObj.hasHttpPar("file_list_id")):
fileListId = reqPropsObj.getHttpPar("file_list_id")
if (reqPropsObj.hasHttpPar("dump_object_info")):
# Dump status of all objects allocated into file specified.
targFile = reqPropsObj.getHttpPar("dump_object_info")
rmFile(targFile)
fo = open(targFile, "w")
fo.write(_genObjectStatus())
fo.close()
# Handle request for file info.
genCfgStatus = 0
genDiskStatus = 0
genFileStatus = 0
genStatesStatus = 1
genRequestStatus = 0
msg = ""
help = 0
if (reqPropsObj.hasHttpPar("help")):
global _help
msg = _help
help = 1
elif hostId and hostId != srvObj.getHostId():
host, port = srvObj.get_remote_server_endpoint(hostId)
cfgObj = srvObj.getCfg()
if not cfgObj.getProxyMode():
httpRef.redirect(host, port)
return
else:
try:
httpRef.proxy_request(hostId, host, port)
except Exception as e:
ex = re.sub("<|>", "", str(e))
errMsg = genLog("NGAMS_ER_COM",
[host, port,ex])
raise Exception(errMsg)
return
elif (fileList):
if (not fileListId):
# It's a new STATUS?file_list request.
fileListId = _handleFileList(srvObj, reqPropsObj, httpRef)
# Send back data from the request.
_handleFileListReply(srvObj, reqPropsObj, httpRef, fileListId,
maxElements)
elif (diskId):
diskObj = ngamsDiskInfo.ngamsDiskInfo()
diskObj.read(srvObj.getDb(), diskId)
status.addDiskStatus(diskObj)
genDiskStatus = 1
elif (fileId):
if (not fileVersion): fileVersion = -1
try:
fileObj = ngamsFileInfo.ngamsFileInfo()
fileObj.read(srvObj.getHostId(), srvObj.getDb(), fileId, fileVersion)
diskObj = ngamsDiskInfo.ngamsDiskInfo()
try:
diskObj.read(srvObj.getDb(), fileObj.getDiskId())
except:
errMsg = "Illegal Disk ID found: {0} for file with ID: {1}".format(fileObj.getDiskId(), fileId)
raise Exception(errMsg)
diskObj.addFileObj(fileObj)
status.addDiskStatus(diskObj)
except:
# The file was not found in the database. Check if it is available
# on a remote partner site.
host, port, status_info, disk_info, file_info = \
ngamsFileUtils.lookup_partner_site_file_status(srvObj, fileId,
fileVersion,
reqPropsObj)
# Update status reply using the disk status and file status
# information retrieved from the partner site
status.addDiskStatus(disk_info)
genDiskStatus = 1
genFileStatus = 1
elif (requestId):
logger.debug("Checking status of request with ID: %s", requestId)
reqPropsObjRef = srvObj.getRequest(requestId)
if (not reqPropsObjRef):
errMsg = genLog("NGAMS_ER_ILL_REQ_ID", [requestId])
raise Exception(errMsg)
genRequestStatus = 1
elif (configurationFile):
msg = "configuration_file=" + srvObj.cfg_fname
genCfgStatus = 1
# Hidden feature to return complete Cfg!!
try:
genCfgStatus = int(configurationFile)
except ValueError:
genCfgStatus = 1
status.setNgamsCfgObj(srvObj.getCfg())
elif (fileAccess):
if (not fileVersion): fileVersion = -1
fileId = fileAccess
msg = _checkFileAccess(srvObj, reqPropsObj, httpRef, fileId,
fileVersion, diskId)
elif (dbTime):
logger.debug("Querying total DB time")
msg = "Total DB time: %.6fs" % srvObj.getDb().getDbTime()
elif (dbTimeReset):
msg = "Resetting DB timer"
logger.debug(msg)
srvObj.getDb().resetDbTime()
else:
msg = "Successfully handled command STATUS"
if (reqPropsObjRef == reqPropsObj):
reqPropsObj.setCompletionTime()
srvObj.updateRequestDb(reqPropsObj)
if (genCfgStatus or genDiskStatus or genFileStatus or genRequestStatus):
status.setReqStatFromReqPropsObj(reqPropsObjRef)
# Generate XML reply.
logger.debug("Status requests: " + str([genCfgStatus, genDiskStatus, genFileStatus,
genStatesStatus]))
xmlStat = status.genXmlDoc(genCfgStatus, genDiskStatus, genFileStatus,
genStatesStatus)
xmlStat = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, xmlStat)
httpRef.send_data(six.b(xmlStat), NGAMS_XML_MT)
elif not httpRef.reply_sent:
httpRef.send_status(msg)
if (msg and (not help)):
logger.info(msg)
else:
logger.info("Successfully handled command STATUS")
def _handleFileListReply(srvObj,
reqPropsObj,
httpRef,
fileListId,
maxElements = None):
"""
Extracts file information from a previously dumped file information
in connection with a STATUS?file_list request.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler object
(ngamsHttpRequestHandler).
fileListId: File List ID allocated to this request (string).
maxElements: Maximum number of elements to extract and return to the
requestor (integer).
Returns: Void.
"""
# Get the name of the DBM in which the information is stored.
dbmBaseName = STATUS_FILE_LIST_DBM_TAG % fileListId
dbmPat = os.path.normpath("%s/*%s*" %\
(ngamsHighLevelLib.\
getNgasTmpDir(srvObj.getCfg()),
dbmBaseName))
dbmMatches = glob.glob(dbmPat)
if (len(dbmMatches) < 1):
msg = "Referenced File List ID: %s in connection with " +\
"STATUS/file_list request, is not (or no longer) known"
raise Exception(msg % fileListId)
elif (len(dbmMatches) > 1):
msg = "Inconsistencies encountered in locating result set for " +\
"STATUS/file_list for referenced File List ID: %s"
raise Exception(msg % fileListId)
fileInfoDbmName = dbmMatches[0]
# Generate the NG/AMS Status Document, with a File List in it.
# Compress it on the fly.
fileListXmlDoc = ngamsHighLevelLib.\
genTmpFilename(srvObj.getCfg(),
"STATUS_FILE_LIST_XML.xml")
try:
tmpFileListDbm = ngamsDbm.ngamsDbm(fileInfoDbmName,
cleanUpOnDestr = 0,
writePerm = 1)
rmFile(fileListXmlDoc)
fo = open(fileListXmlDoc, "w")
if (not maxElements):
remainingObjects = 0
else:
remainingObjects = (tmpFileListDbm.getCount() - maxElements)
if (remainingObjects < 0):
remainingObjects = 0
fo.write(_fileListXmlHdr % (fileListId, str(remainingObjects)))
# Loop over the file info objects and write them into the file.
# take max the number of elements specified.
tmpFileListDbm.initKeyPtr()
elCount = 0
keyRefList = []
while (True):
# Have the requested number of elements been extracted?
if (maxElements):
if (elCount >= maxElements):
break
# Get the next key (if there are more elements).
key, fileInfo = tmpFileListDbm.getNext()
if (not key):
break
try:
# Write the File Status XML Element in the file.
tmpFileInfoObj = ngamsFileInfo.ngamsFileInfo().\
unpackSqlResult(fileInfo)
fileInfoXml = tmpFileInfoObj.genXml(storeDiskId = 1).\
toprettyxml(" ", "\n")[:-1]
fo.write("\n" + fileInfoXml)
except Exception as e:
msg = "Error creating STATUS/File List XML Document. " +\
"Error: %s" % str(e)
logger.error(msg)
raise
keyRefList.append(key)
elCount += 1
# Finish up the XML document, close the file.
fo.write(_fileListXmlFooter)
fo.close()
# Assume this type of file can always be compressed.
fileListXmlDoc = compressFile(fileListXmlDoc)
except:
rmFile("%s*" % fileInfoDbmName)
rmFile("%s*" % fileListXmlDoc)
raise
# Send the XML document back to the requestor.
try:
httpRef.send_file(fileListXmlDoc, NGAMS_GZIP_XML_MT)
# Remove the reported entries.
for key in keyRefList:
tmpFileListDbm.rem(key)
tmpFileListDbm.sync()
del keyRefList
keyRefList = []
# If there are no more entries, delete the DBM.
dbmCount = tmpFileListDbm.getCount()
del tmpFileListDbm
if (dbmCount == 0):
rmFile("%s*" % fileInfoDbmName)
except Exception as e:
msg = "Error returning response to STATUS?file_list request. Error: %s"
msg = msg % str(e)
raise Exception(msg)
finally:
rmFile(fileListXmlDoc)
def _remFile(srvObj, reqPropsObj, diskId, fileId, fileVersion, execute,
tmpFilePat):
"""
See documentation for the ngamsRemFileCmd.remFile() function.
"""
# Check for illegal parameter combinations.
if ((not diskId) or (diskId and (not fileId))):
errMsg = "Disk ID: %s, File ID: %s, File Version: %d" %\
(diskId, fileId, fileVersion)
errMsg = genLog("NGAMS_ER_CMD_SYNTAX", [NGAMS_REMFILE_CMD, errMsg])
raise Exception(errMsg)
# Temporary DBM to contain SQL info about files concerned by the query.
fileListDbmName = os.path.normpath(tmpFilePat + "_FILE_LIST")
fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm=1)
# Get the information from the DB about the files in question.
hostId = None
diskIds = []
fileIds = []
if (diskId):
diskIds = [diskId]
elif ((not diskId) and fileId):
hostId = srvObj.getHostId()
if (fileId): fileIds = [fileId]
if (fileVersion == -1): fileVersion = None
n_files = 0
for f in srvObj.db.getFileSummary1(hostId, diskIds, fileIds, ignore=None):
if fileVersion is not None and fileVersion != f[
ngamsDbCore.SUM1_VERSION]:
continue
msg = "Scheduling file with ID: %s/%d on disk with ID: %s for " +\
"deletion"
logger.debug(msg, f[ngamsDbCore.SUM1_FILE_ID],
f[ngamsDbCore.SUM1_VERSION], f[ngamsDbCore.SUM1_DISK_ID])
fileListDbm.add(str(n_files), f)
n_files += 1
fileListDbm.sync()
# Check if the files selected for deletion are available within the NGAS
# system, in at least 3 copies.
filesMisCopyDbmName, filesNotRegDbmName, complFileListDbmName =\
ngamsRemUtils.checkFileCopiesAndReg(srvObj,
3, tmpFilePat,
fileListDbmName)
status = ngamsRemUtils._remStatErrReport(srvObj, reqPropsObj, tmpFilePat,
filesMisCopyDbmName,
filesNotRegDbmName,
fileListDbmName, diskId, fileId,
fileVersion)
if (status): return status
# Check that none of the matching files are stored on other NGAS Nodes.
# If such are found, do not consider these.
fileListDbm.initKeyPtr()
remKeyList = []
while (1):
key, fileInfo = fileListDbm.getNext()
if (not key): break
if (fileInfo[ngamsDbCore.SUM1_HOST_ID] != srvObj.getHostId()):
remKeyList.append(key)
for remKey in remKeyList:
fileListDbm.rem(remKey)
#########################################################################
# Execute the deletion if execute = 1 and files were found to be deleted.
#########################################################################
successDelCount = 0
failedDelCount = 0
if (execute):
fileListDbm.initKeyPtr()
run = 1
# TODO: This should be changed to a single or a few DB transactions for all files
# and a bulk rm for the same number of files.
while (run):
key, fileInfo = fileListDbm.getNext()
if (not key):
run = 0
continue
try:
diskId = fileInfo[ngamsDbCore.SUM1_DISK_ID]
fileId = fileInfo[ngamsDbCore.SUM1_FILE_ID]
fileVer = fileInfo[ngamsDbCore.SUM1_VERSION]
mtPt = fileInfo[ngamsDbCore.SUM1_MT_PT]
filename = fileInfo[ngamsDbCore.SUM1_FILENAME]
complFilename = os.path.normpath(mtPt + "/" + filename)
msg = "Deleting DB info for file: %s/%s/%d"
logger.debug(msg, diskId, fileId, fileVer)
# We remove first the DB info and afterwards the file on the
# disk. The reason for this is that it is considered worse
# to have an entry for a file in the DB, which is not on disk
# than vice versa, since NGAS uses the info in the DB to check
# for the number of available copies.
try:
srvObj.getDb().deleteFileInfo(srvObj.getHostId(), diskId,
fileId, fileVer)
infoMsg = genLog("NGAMS_INFO_DEL_FILE",
[diskId, fileId, fileVer])
logger.debug(infoMsg)
successDelCount += 1
except Exception, e:
failedDelCount += 1
errMsg = genLog("NGAMS_ER_DEL_FILE_DB",
[diskId, fileId, fileVer,
str(e)])
logger.warning(errMsg)
# Removing the DB info was successful, remove the copy on disk.
msg = "Deleting copy of file: %s/%s/%d: %s"
logger.debug(msg, diskId, fileId, fileVer, complFilename)
rmFile(complFilename)
except Exception, e:
failedDelCount += 1
errMsg = genLog(
"NGAMS_ER_DEL_FILE_DISK",
[diskId, fileId, fileVer, str(e)])
logger.warning(errMsg)