def _restoreSubscriptionInfoFromDisk(srvObj):
ngas_root_dir = srvObj.getCfg().getRootDirectory()
myDir = ngas_root_dir + "/SubscriptionInfo"
if (not os.path.exists(myDir)):
return
saveFile = myDir + "/SubscriptionInfoObj"
if (not os.path.exists(saveFile)):
return
logger.debug('Restoring subscription info from disks ...')
saveObj = None
try:
pkl_file = open(saveFile, 'rb')
saveObj = pickle.load(pkl_file)
pkl_file.close()
except Exception:
logger.exception('Fail to restore subscription info from disks')
if (saveObj == None or len(saveObj) == 0):
return
logger.debug(
'Appending subscription info to info list kept internally ...')
try:
srvObj._subscriptionSem.acquire()
srvObj._subscriptionFileList += saveObj
cmd = "rm " + saveFile
ngamsPlugInApi.execCmd(cmd, -1)
except Exception:
logger.exception('Fail to append filelist to subscription info list')
finally:
srvObj._subscriptionSem.release()
def stageFiles(filenameList):
"""
Stage a list of files.
The system will sort files in the order that they are archived on the tape volumes for better performance
"""
cmd1 = "stage -r"
cmd2 = "stage -r -w"
num_staged = 0
for filename in filenameList:
if (isFileOnTape(filename) == 1):
cmd1 = cmd1 + " " + filename
cmd2 = cmd2 + " " + filename
num_staged = num_staged + 1
t = ngamsPlugInApi.execCmd(cmd1, -1)
exitCode = t[0]
if (exitCode != 0):
logger.error("Staging problem: %s, cmd: %s", str(exitCode), cmd1)
return -1
t = ngamsPlugInApi.execCmd(cmd2, -1)
exitCode = t[0]
if (exitCode != 0):
logger.error("Staging problem: %s, cmd: %s", str(exitCode), cmd2)
return -1
return num_staged
def checkFitsChecksum(reqPropsObj, stgFile=None):
"""
Carry out a check of the DATASUM and CHECKSUM for each HDU in the file.
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
stgFile: If specified this is taken rather than from the Request
Properties Object (ngamsReqProps).
Returns: Void.
"""
if (not stgFile): stgFile = reqPropsObj.getStagingFilename()
cmd = "chksumVerFitsChksum %s" % stgFile
stat, out = ngamsPlugInApi.execCmd(cmd)
if (out.find("Status: OK") == -1):
if (out.find("Status: NOT OK") != -1):
errMsg = genLog(
"NGAMS_ER_DAPI_BAD_FILE",
[stgFile, "checkFitsChecksum", "Illegal CHECKSUM/DATASUM"])
else:
errMsg = "Problem found while checking file: %s. " %\
out.replace("\n", " ")
# Backwards compatibility with old ESO FITS CHECKSUM scheme: If the
# file pass the previous scheme, we consider it as OK and do not
# return the failure.
cmd = "chksumGenChecksum %s" % stgFile
stat, out = ngamsPlugInApi.execCmd(cmd)
if (out.strip().find("0/0000000000000000") == -1):
raise Exception, errMsg
chksumUtil = "chksumGenChecksum"
else:
chksumUtil = "chksumVerFitsChksum"
logger.debug("File: %s checked with: %s. Result: OK", stgFile, chksumUtil)
def execCmd(cmd, failonerror=True, okErr=[]):
re = ngamsPlugInApi.execCmd(cmd)
if (re[0] != 0 and not (re[0] in okErr)):
errMsg = 'Fail to execute command: "%s". Exception: %s' % (cmd, re[1])
if (failonerror):
raise Exception(errMsg)
else:
print(errMsg)
return re
def execCmd(cmd, timeout):
logger.debug('Executing command: %s', cmd)
try:
ret = ngamsPlugInApi.execCmd(cmd, timeout)
except Exception, ex:
if (str(ex).find('timed out') != -1):
return (-1, 'Timed out (%d seconds): %s' % (timeout, cmd))
else:
return (-1, str(ex))
def _saveSubscriptionInfoToDisk(srvObj):
"""
Before ngas server is shutdown, write files from to-be-delivered-list to disks
otherwise, this information will be lost when server is restarted again
"""
# get the files currently being transferred but not yet completed
# save them in the subscriptionInfo so that they can be resumed when server starts up
li = []
idx_fileId = ngamsSubscriptionThread.FILE_ID
idx_filever = ngamsSubscriptionThread.FILE_VER
for k, fileInfo in srvObj._subscrDeliveryFileDic.items():
if (fileInfo == None):
continue
fileInfo = ngamsSubscriptionThread._convertFileInfo(fileInfo)
li += [(fileInfo[idx_fileId], fileInfo[idx_filever])]
logger.debug('%s is added to the subscriptionInfoList',
fileInfo[idx_fileId])
# also add to files that have not yet been in the queue
srvObj._subscriptionSem.acquire()
srvObj._subscriptionFileList += li
if (len(srvObj._subscriptionFileList) == 0):
srvObj._subscriptionSem.release()
logger.debug("**** subscription list is empty!!")
return
logger.debug("Saving subscription info to disks ...")
ngas_root_dir = srvObj.getCfg().getRootDirectory()
myDir = ngas_root_dir + "/SubscriptionInfo"
saveFile = myDir + "/SubscriptionInfoObj"
try:
if (os.path.exists(saveFile)):
cmd = "rm " + saveFile
ngamsPlugInApi.execCmd(cmd, -1)
if (not os.path.exists(myDir)):
os.makedirs(myDir)
output = open(saveFile, 'wb')
pickle.dump(srvObj._subscriptionFileList, output)
output.close()
except Exception:
logger.exception('Fail to save subscription info to disks')
finally:
srvObj._subscriptionSem.release()
def stopAsyncQService(srvObj, reqPropsObj):
"""
# when the server is shutdown, this is called to stop threads, and save uncompleted list back to database
"""
ngas_root_dir = srvObj.getCfg().getRootDirectory()
myDir = ngas_root_dir + "/AsyncQService"
saveFile = myDir + "/AsyncRetrieveListObj"
uuids = asyncReqDic.keys()
if (len(uuids) == 0):
if (os.path.exists(saveFile)):
cmd = "rm " + saveFile
ngamsPlugInApi.execCmd(cmd, -1)
return "ok after deleting the file"
for sessionId in uuids:
suspendHandler(srvObj, reqPropsObj, sessionId)
#info(3, "Stopping - root dir = %s" % ngas_root_dir)
if (not os.path.exists(myDir)):
os.makedirs(myDir)
"""
asyncReqDic = {} #key - uuid, value - AsyncListRetrieveRequest (need to remember the original request in case of cancel/suspend/resume or server shutting down)
statusResDic = {} #key - uuid, value - AsyncListRetrieveStatusResponse
nextFileDic = {} #key - uuid, value - next file_id to be delivered
threadDic = {} #key - uuid, value - the threadref
threadRunDic = {} #key - uuid, value - 1/0, 1: run 0: stop
"""
saveObj = [asyncReqDic, statusResDic, nextFileDic]
try:
output = open(saveFile, 'wb')
pickle.dump(saveObj, output)
output.close()
except Exception as e:
ex = str(e)
return ex
return "ok"
def ngamsMWACortexStageDppi(srvObj,
reqPropsObj,
filename):
"""
This plugin stages the file from Tape to the disk, works ONLY in the iVEC Cortex environment.
srvObj: Reference to instance of the NG/AMS Server
class (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
filename: Name of file to process (string).
Returns: DPPI return status object (ngamsDppiStatus).
"""
#mimeType = ngamsPlugInApi.determineMimeType(srvObj.getCfg(), filename)
mimeType = "application/octet-stream" #hardcode this since our file extension is actually ".fits"
resultObj = ngamsDppiStatus.ngamsDppiResult(NGAMS_PROC_FILE, mimeType,
filename, filename)
statusObj = ngamsDppiStatus.ngamsDppiStatus().addResult(resultObj)
#procFilename, procDir = ngamsPlugInApi.prepProcFile(srvObj.getCfg(), filename)
cmd = "sls -D " + filename
t = ngamsPlugInApi.execCmd(cmd, -1)
exitCode = t[0]
if (exitCode != 0 or len(t) != 2):
logger.error("Fail to query the online/offline status for file %s", filename)
return statusObj #the sls -D command failed to execute, but retrieval might still go on, so just simply return empty result
offline = t[1].find('offline;')
if (offline != -1): # the file is offline, i.e. it is on tape
logger.debug("File %s is currently on tapes, staging it for retrieval...", filename)
cmd = "stage -w " + filename
t = ngamsPlugInApi.execCmd(cmd, -1) #stage it back to disk cache
logger.debug("File %s staging completed.", filename)
return statusObj
def ngamsMWA_Compress_FilterPlugin(srvObj,
plugInPars,
filename,
fileId,
fileVersion=-1,
reqPropsObj=None):
"""
srvObj: Reference to NG/AMS Server Object (ngamsServer).
plugInPars: Parameters to take into account for the plug-in
execution (string).
fileId: File ID for file to test (string).
filename: Filename of (complete) (string).
fileVersion: Version of file to test (integer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: 0 if the file does not match, 1 if it matches the
conditions (integer/0|1).
"""
if (not isMWAVisFile(fileId)):
return 0 # only add MWA Vis FITS file
cmd = 'head -c %d %s' % (1024 * 3, filename)
try:
re = ngamsPlugInApi.execCmd(cmd)
except Exception as ex:
if (str(ex).find('timed out') != -1):
logger.error('Timed out when checking FITS header %s', cmd)
else:
logger.error('Exception when checking FITS header %s: %s', cmd,
str(ex))
return 0
if (0 == re[0]):
a = re[1].find("XTENSION= 'BINTABLE'")
if (a > -1):
return 0 # if the file is already compressed, do not add again
else:
logger.info("File %s added", filename)
return 1
else:
logger.warning('Fail to check header for file %s: %s', filename, re[1])
return 0
def isFileOnTape(filename):
"""
return 1 - on tape, 0 - not on tape, -1 - query error
"""
cmd = "sls -D " + filename
t = ngamsPlugInApi.execCmd(cmd, -1)
exitCode = t[0]
if (exitCode != 0 or len(t) != 2):
logger.error("Fail to query the online/offline status for file %s",
filename)
#print errMsg
return -1 #raise error
offline = t[1].find(
'offline;'
) # Do not use "archdone" any more given this reason: http://www.mail-archive.com/[email protected]/msg00419.html
if (offline != -1): # the file is offline, i.e. it is on tape
return 1
else:
return 0
def stageFile(filename):
cmd = "stage -w " + filename
print("File %s is on tape, staging it now..." % filename)
ngamsPlugInApi.execCmd(cmd, -1) #stage it back to disk cache
print("File " + filename + " staging completed.")
def compressFile(srvObj, reqPropsObj, parDic):
"""
Compress the file if required.
srvObj: Reference to NG/AMS Server Object (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
parDic: Dictionary with parameters for the DAPI. This is generated
with ngamsPlugInApi.parseDapiPlugInPars() (Dictionary).
Returns: Tupe containing uncompressed filesize, archived filesize
and the format (mime-type) of the resulting data file and
the compression method (NONE if the file is not compressed),
finally, the extension added by the compression if any
(tuple).
"""
stFn = reqPropsObj.getStagingFilename()
# If a compression application is specified, apply this.
uncomprSize = ngamsPlugInApi.getFileSize(stFn)
comprExt = ""
if (parDic[COMPRESSION]):
logger.debug("Compressing file using: %s ...", parDic[COMPRESSION])
compCmd = "%s %s" % (parDic[COMPRESSION], stFn)
compress_start = time.time()
logger.debug("Compressing file with command: %s", compCmd)
exitCode, stdOut = ngamsPlugInApi.execCmd(compCmd)
#if (exitCode != 0):
# msg ="Problems during archiving! Compressing the file failed. " +\
# "Error: %s" % str(stdOut).replace("/n", " ")
# raise Exception, msg
# If the compression fails, assume that it is because the file is not
# compressible (although it could also be due to lack of disk space).
if (exitCode == 0):
if (parDic[COMPRESSION_EXT]):
stFn = stFn + "." + parDic[COMPRESSION_EXT]
comprExt = parDic[COMPRESSION_EXT]
# Remember to update Staging Filename in the Request Properties
# Object.
reqPropsObj.setStagingFilename(stFn)
# Handle mime-type
if (parDic[TARG_MIME_TYPE]):
format = parDic[TARG_MIME_TYPE]
else:
format = ngamsPlugInApi.determineMimeType(
srvObj.getCfg(), stFn)
compression = parDic[COMPRESSION]
logger.debug("File compressed. Time: %.3fs",
time.time() - compress_start)
else:
# Carry on with the original file. We take the original mime-type
# as the target mime-type.
format = reqPropsObj.getMimeType()
compression = NO_COMPRESSION
else:
# Handle mime-type
if (parDic[TARG_MIME_TYPE]):
format = parDic[TARG_MIME_TYPE]
else:
format = reqPropsObj.getMimeType()
compression = NO_COMPRESSION
archFileSize = ngamsPlugInApi.getFileSize(reqPropsObj.getStagingFilename())
return uncomprSize, archFileSize, format, compression, comprExt
def ngamsBrotherPT9200DxPlugIn(srvObj, label, reqPropsObj=None):
"""
Driver for printing labels on the label printer Brother PT-9200DX.
srvObj: Reference to instance of the NG/AMS Server
class (ngamsServer).
label: Label text to print (string).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: Void.
"""
plugInPars = srvObj.getCfg().getLabelPrinterPlugInPars()
logger.info("Executing plug-in ngamsBrotherPT9200DxPlugIn with parameters: " + \
"%s - Label: %s ...", plugInPars, label)
parDic = ngamsPlugInApi.parseRawPlugInPars(plugInPars)
# Get the font bit pattern dictionary.
fontDic = genFontsDictionary(parDic["font_file"])
# Generate the printer control code.
printerCode = fontDic["Header"]
for i in range(len(label)):
if (not fontDic.has_key(label[i])):
errMsg = "No font defintion for character: \"" + label[i] +\
"\" - in font definition file: " + parDic["font_file"] +\
" - cannot generate disk label: " + label
ngamsPlugInApi.notify(srvObj, NGAMS_NOTIF_ERROR,
"ngamsBrotherPT9200DxPlugIn: " +\
"ILLEGAL CHARACTER REQ. FOR PRINTING",
errMsg)
raise Exception(errMsg)
printerCode = printerCode + fontDic[label[i]]
printerCode = printerCode + fontDic["Trailer"]
# Generate printer file, write the printer control code.
tmpDir = ngamsPlugInApi.getTmpDir(srvObj.getCfg())
ngasId = srvObj.getHostId()
printerFilename = os.path.normpath(tmpDir + "/ngamsLabel_" + ngasId +
".prn")
fo = open(printerFilename, "w")
fo.write(printerCode)
fo.close()
# Write the printer code file to the device.
stat, out = ngamsPlugInApi.execCmd("cat " + printerFilename + " > " +\
parDic["dev"])
# This was previously excluded during the "test mode"; a proper way is by
# having a configurable parameter
if not 'keep_printer_file' in parDic:
os.system("rm -f " + printerFilename)
if (stat != 0):
errMsg = "Problem occurred printing label! Error: " + str(out)
ngamsPlugInApi.notify(srvObj, NGAMS_NOTIF_ERROR,
"ngamsBrotherPT9200DxPlugIn: " +\
"PROBLEM PRINTING LABEL", errMsg)
raise Exception(errMsg)
def ngamsFitsRegPlugIn(srvObj, reqPropsObj, parDic):
"""
Data Registration Plug-In to handle registration of FITS files.
srvObj: Reference to NG/AMS Server Object (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: Standard NG/AMS Data Archiving Plug-In Status as generated
by: ngamsPlugInApi.genDapiSuccessStat() (ngamsDapiStatus).
"""
logger.info("Plug-In registering file with URI: %s", reqPropsObj.getFileUri())
diskInfo = reqPropsObj.getTargDiskInfo()
stageFile = reqPropsObj.getStagingFilename()
# If the file is already compressed, we have to decompress it.
procDir = ""
if ((stageFile.find(".Z") != -1) or (stageFile.find(".gz") != -1)):
workingFile, procDir = ngamsPlugInApi.prepProcFile(srvObj.getCfg(),
stageFile)
ngamsPlugInApi.execCmd("gunzip " + workingFile)
if (workingFile.find(".Z") != -1):
workingFile = workingFile[:-2]
else:
workingFile = workingFile[:-3]
else:
workingFile = stageFile
# Check file (size + checksum).
ngamsFitsPlugIn.checkFitsFileSize(workingFile)
#ngamsFitsPlugIn.c_heckChecksum(parDic, workingFile)
if 'skip_checksum' not in parDic:
ngamsFitsPlugIn.checkFitsChecksum(reqPropsObj, workingFile)
# Get various information about the file being handled.
arcFile, dpId, dateDirName = ngamsFitsPlugIn.getDpIdInfo(workingFile)
fileVersion, relPath, relFilename,\
complFilename, fileExists =\
ngamsPlugInApi.genFileInfoReg(srvObj.getDb(), srvObj.getCfg(),
reqPropsObj, diskInfo,
stageFile, dpId)
# Generate status.
logger.debug("Generating status ...")
fileSize = ngamsPlugInApi.getFileSize(stageFile)
if (stageFile.find(".Z") != -1):
format = "application/x-cfits"
compresion = "compress"
elif (stageFile.find(".gz") != -1):
format = "application/x-gfits"
compresion = "gzip"
else:
format = "image/x-fits"
compresion = ""
uncomprSize = ngamsPlugInApi.getFileSize(workingFile)
# Delete the processing directory (would be done later by the
# Janitor Thread, but it is better to clean up explicitly).
if (procDir): rmFile(procDir)
logger.debug("Register Plug-In finished processing of file")
return ngamsPlugInApi.genRegPiSuccessStat(diskInfo.getDiskId(),relFilename,
dpId, fileVersion, format,
fileSize, uncomprSize,compresion,
relPath, diskInfo.getSlotId(),
fileExists, complFilename)
def ngamsNgLogPlugIn(srvObj, reqPropsObj):
"""
Data Archiving Plug-In to handle archiving of NG/AMS (OLAS style) log
files.
srvObj: Reference to NG/AMS Server Object (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: Standard NG/AMS Data Archiving Plug-In Status
as generated by: ngamsPlugInApi.genDapiSuccessStat()
(ngamsDapiStatus).
"""
# For now the exception handling is pretty basic:
# If something goes wrong during the handling it is tried to
# move the temporary file to the Bad Files Area of the disk.
logger.debug("Plug-In handling data for file: %s",
os.path.basename(reqPropsObj.getFileUri()))
diskInfo = reqPropsObj.getTargDiskInfo()
stagingFilename = reqPropsObj.getStagingFilename()
ext = os.path.splitext(stagingFilename)[1][1:]
# Now, build final filename. We do that by taking the date of
# the first log entry.
#
# 2001-07-09T14:37:59.563 [INFO] Logging properties defined ...
#
# Alternatively the first entry could be of the form, e.g.:
#
# 2003-12-29T09:21:57.608 [INFO] LOG-ROTATE: 1072689717 - \
# SYSTEM-ID: ngamsArchiveClient@ngasdev2
#
# In the former case the Log ID is equal to the NGAS ID. In the latter
# case, the Log ID is equal to the System ID.
#
# The final filename is built as follows: <Log ID>.<date>.<ext>
#
# The file_id is: <Log ID>.<date>
fo = open(stagingFilename, "r")
firstLine = fo.readline()
fo.close()
try:
# Compress the log file.
uncomprSize = ngamsPlugInApi.getFileSize(stagingFilename)
compression = "gzip"
logger.debug("Compressing file using: %s ...", compression)
exitCode, stdOut = ngamsPlugInApi.execCmd("%s %s" %\
(compression,
stagingFilename))
if (exitCode != 0):
errMsg = "ngamsNgLogPlugIn: Problems during archiving! " +\
"Compressing the file failed"
raise Exception(errMsg)
stagingFilename = stagingFilename + ".gz"
# Remember to update the Temporary Filename in the Request
# Properties Object.
reqPropsObj.setStagingFilename(stagingFilename)
logger.debug("Log file compressed")
# Parse first line of the log file.
timeStamp = firstLine.split(" ")[0]
date = timeStamp.split("T")[0]
sysIdIdx = firstLine.find("SYSTEM-ID")
if (sysIdIdx != -1):
logId = firstLine[sysIdIdx + len("SYSTEM-ID:"):].strip().\
split(" ")[0]
else:
logId = srvObj.getHostId()
fileId = logId + "." + timeStamp
fileVersion, relPath, relFilename,\
complFilename, fileExists =\
ngamsPlugInApi.genFileInfo(srvObj.getDb(),
srvObj.getCfg(),
reqPropsObj, diskInfo,
stagingFilename, fileId,
fileId, [date])
# Generate status.
logger.debug("Generating status ...")
fformat = ngamsPlugInApi.determineMimeType(srvObj.getCfg(),
stagingFilename)
logger.debug("Determining file size ...")
fileSize = ngamsPlugInApi.getFileSize(stagingFilename)
return ngamsPlugInApi.genDapiSuccessStat(
diskInfo.getDiskId(), relFilename, fileId, fileVersion,
fformat, fileSize, uncomprSize, compression, relPath,
diskInfo.getSlotId(), fileExists, complFilename)
except Exception:
raise Exception("ngamsNgLogPlugIn: Error handling log file: " +\
stagingFilename + ". Rejecting.")
def execCmd(cmd, failonerror=True):
re = ngamsPlugInApi.execCmd(cmd)
if (failonerror and (not os.WIFEXITED(re[0]))):
errMsg = 'Fail to execute command: "%s". Exception: %s' % (cmd, re[1])
raise Exception(errMsg)
return re
