def ngamsExampleCacheCtrlPI(srvObj, cacheEntryObj):
"""
Example Cache Control Plug-in to control the cache holding based on
an expiration time for each cached file.
srvObj: Reference to NG/AMS Server Object (ngamsServer).
cacheEntryObj: Cache Entry Object containing the information for the
cached file (ngamsCacheEntry).
Returns: Returns True if the file can be deleted from the cache,
otherwise False (boolean).
"""
T = TRACE()
try:
global ngamsExampleCacheCtrlPI_maxCacheTime
if (not ngamsExampleCacheCtrlPI_maxCacheTime):
plugInPars = srvObj.getCfg().\
getVal("Caching[1].CacheControlPlugInPars")
plugInParDic = ngamsPlugInApi.parseRawPlugInPars(plugInPars)
try:
_maxCacheTime = float(plugInParDic["max_cache_time"])
except:
msg = "Missing plug-in parameter: max_cache_time"
raise Exception, msg
# Check if the cache time is exceeded.
if ((time.time() - cacheEntryObj.getCacheTime()) > _maxCacheTime):
return True
return False
except Exception:
raise
def ngamsGLEAM_Phase_FilterPI(srvObj,
plugInPars,
filename,
fileId,
fileVersion = -1,
reqPropsObj = None):
"""
srvObj: Reference to NG/AMS Server Object (ngamsServer).
plugInPars: Parameters to take into account for the plug-in
execution (string).
fileId: File ID for file to test (string).
filename: Filename of (complete) (string).
fileVersion: Version of file to test (integer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: 0 if the file does not match, 1 if it matches the
conditions (integer/0|1).
"""
if (not isGLEAMImage(fileId)):
return 0
if (not srvObj.getDb().isLastVersion(fileId, fileVersion)):
return 0
pars = ""
if ((plugInPars != "") and (plugInPars != None)):
pars = plugInPars
elif (reqPropsObj != None):
if (reqPropsObj.hasHttpPar("plug_in_pars")):
pars = reqPropsObj.getHttpPar("plug_in_pars")
parDic = ngamsPlugInApi.parseRawPlugInPars(pars)
if (parDic.has_key('phase')):
phase = int(parDic['phase'])
else:
return 1 # no need to check phase
img_phase = None
try:
img_phase = getGLEAMPhase(filename)
except Exception as exp:
logger.warning("cannot get phase info from %s, file not added: %s" % (filename, str(exp)))
return 0
if (phase == img_phase):
return 1
else:
return 0
def ngamsMimeTypeFilterPI(srvObj,
plugInPars,
filename,
fileId,
fileVersion=-1,
reqPropsObj=None):
"""
Example Filter Plug-In used to filter on a given mime-type. In case the
file referenced has the mime-type as specified in the plug-in parameters,
the file being tested is selected.
srvObj: Reference to NG/AMS Server Object (ngamsServer).
plugInPars: Parameters to take into account for the plug-in
execution (string).
fileId: File ID for file to test (string).
filename: Filename of (complete) (string).
fileVersion: Version of file to test (integer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: 0 if the file does not match, 1 if it matches the
conditions (integer/0|1).
"""
match = 0
# Parse plug-in parameters.
parDic = []
pars = ""
if ((plugInPars != "") and (plugInPars != None)):
pars = plugInPars
elif (reqPropsObj != None):
if (reqPropsObj.hasHttpPar("plug_in_pars")):
pars = reqPropsObj.getHttpPar("plug_in_pars")
parDic = ngamsPlugInApi.parseRawPlugInPars(pars)
if (not parDic.has_key("mime_types")):
errMsg = "ngamsMimeTypeFilterPI: Missing Plug-In Parameter: " +\
"mime_types"
raise Exception, errMsg
# Perform the matching.
refMimeTypes = parDic["mime_types"].split("|")
actMimeType = ngamsPlugInApi.determineMimeType(srvObj.getCfg(), filename)
for mt in refMimeTypes:
if (actMimeType == mt.strip()): match = 1
return match
def ngamsAlmaCacheCtrlPI(srvObj, cacheEntryObj):
"""
srvObj: Reference to NG/AMS Server Object (ngamsServer).
cacheEntryObj: Cache Entry Object containing the information for the
cached file (ngamsCacheEntry).
Returns: Returns True if the file can be deleted from the cache,
otherwise False (boolean).
"""
plugInPars = srvObj.getCfg().getVal("Caching[1].CacheControlPlugInPars")
plugInParDic = ngamsPlugInApi.parseRawPlugInPars(plugInPars)
dblinks = plugInParDic["db_links_list"]
dblinks_list = dblinks.split(":")
for dblink in dblinks_list:
query = "select count(*) from ngas_files@" + dblink
srvObj.getDb().query2(query)
def ngamsExtractAlmaMultipart(srvObj, reqPropsObj, file_id):
"""
This DPPI extracts one part of a multipart/related message
requested from the ALMA Archive.
srvObj: Reference to instance of the NG/AMS Server
class (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
file_id: Name of file to process (string).
Returns: DPPI return status object (ngamsDppiStatus).
Side effect: This DPPI works directly on the archived file, since it is
read-only access.
SPECIFIC DOCUMENTATION:
This DPPI extracts one part of an ALMA multipart related message and
returns a complete, self-consistent message containing a XML header
and the requested cid. This version deals with VOTable headers and
returns a new VOTable header containing only the RESOURCE element
describing the requested (by cid) part. If no XML header can be found
or if the XML header is not a VOTable header this plugin returns just the
requested part, without any header.
Example URL (single line):
http://ngasdev1:7777/RETRIEVE
?file_id=X01/X7/X42&
processing=ngamsAlmaMultipart&
processing_pars='cid=<cid>'
"""
statusObj = ngamsDppiStatus.ngamsDppiStatus()
cpart = 1
pars = 0 # initialize pars
if (reqPropsObj.hasHttpPar("processing_pars")):
pars = ngamsPlugInApi.parseRawPlugInPars(\
reqPropsObj.getHttpPar("processing_pars"))
if pars and not pars.has_key('cid'):
ext = '.cid'
cpart = 1 # first part only by default
elif pars and pars.has_key(
'cid'): # if processing_par 'cid' exists check its contents
pass
else:
pass
resFilename = file_id + ext
try:
mimeType = ngamsPlugInApi.determineMimeType(srvObj.getCfg(),\
resFilename)
except:
pass
if ext == '.xml':
mimeType = 'text/xml'
else:
mimeType = 'multipart/related'
resObj = ngamsDppiStatus.ngamsDppiResult(NGAMS_PROC_DATA, mimeType, head,
resFilename, '')
statusObj.addResult(resObj)
return statusObj
def _registerExec(srvObj,
fileListDbmName,
tmpFilePat,
diskInfoDic,
reqPropsObj=None):
"""
Register the files listed in the File List DBM (ngamsDbm), which
match the mime-type(s) either specified in the 'mimeType' parameter,
or if this is not specified, which match all the mime-types specified
in the configuration file.
When the registration procedure has been executed, the function sends
an Email Notification message indicating which files were registered
if the HTTP parameter 'notif_email' is given.
The functions creates a File Info Objects per file handled and
writes this in a temporary file, which is a DBM file. The
keys in this DB is simply the file number in the sequence of files
handled, pointing to a pickled ngamsFileInfo object.
Each of the File Info Objects indicates if the file was registered or not.
This is done by setting the tag of the File Info Object to one of the
following values:
REGISTERED: The file was successfully registered in the NGAS DB
FAILED: The file was selected for registration but could not
be properly registered because of inconsistencies.
The status will be of the format: 'FAILED[: <reason>]'.
REJECTED: A file found under the specified path directory was
not accepted for cloning, usually because the mime-type
was not correct. The status will be of the format:
'REJECTED[: <reason>]'.
Note, that registration is comparable to archiving of files. For that
reason a DAPI must be provided for each type of file that should be
registered. If this is not fullfilled, the file registration will
fail.
Only files stored on one of the NGAS disks configured in the
configuration file, are considered. Files stored in other locations
are ignored.
srvObj: Instance of NG/AMS Server object (ngamsServer).
fileListDbmName: Name of a DBM containing the information
about the files to be registered. Each element in the
list is referred to by a key, which is a number.
These points to a pickled list for each file containing
the following information:
[<Filename>, <Disk ID>, <Mime-Type>]
The information for each disk concerned is also contained
in the DB referred to by its Disk ID and by its mount
point. The data is a pickled instance of the ngamsDiskInfo
class (string).
tmpFilePat: Pattern for temporary files used during the registration
process (string).
diskInfoDic: Dictionary with Disk IDs as keys pointing to the info
about the disk (dictionary/ngamsDiskInfo).
reqPropsObj: If an NG/AMS Request Properties Object is given, the
Request Status will be updated as the request is carried
out (ngamsReqProps).
Returns: Void.
"""
emailNotif = 0
if (reqPropsObj):
if (reqPropsObj.hasHttpPar("notif_email")):
emailNotif = 1
# Create the temporary BSD DB to contain the information for the
# Email Notification Message.
if (emailNotif):
regDbmName = tmpFilePat + "_NOTIF_EMAIL"
regDbm = ngamsDbm.ngamsDbm(regDbmName, writePerm=1)
# Open the DBM containing the list of files to (possibly) register.
fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm=1)
# Want to parse files in alphabetical order.
# TODO: Portatibility issue. Try to avoid UNIX shell commands for sorting.
tmpFileList = tmpFilePat + "_FILE_LIST"
rmFile(tmpFileList)
with open(tmpFileList, "wb") as fo:
fileListDbm.initKeyPtr()
while (1):
dbmKey, fileInfo = fileListDbm.getNext()
if (not dbmKey): break
fo.write(dbmKey + b"\n")
sortFileList = tmpFilePat + "_SORT_FILE_LIST"
rmFile(sortFileList)
shellCmd = "sort %s > %s" % (tmpFileList, sortFileList)
stat, out, err = ngamsCore.execCmd(shellCmd)
if (stat != 0):
raise Exception("Error executing command: %s. Error: %s, %s" %\
(shellCmd, str(out), str(err)))
rmFile(tmpFileList)
# Go through each file in the list, check if the mime-type is among the
# ones, which apply for registration. If yes try to register the file
# by invoking the corresponding DAPI on the file.
fileRegCount = 0
fileFailCount = 0
fileRejectCount = 0
regTimeAccu = 0.0
fileCount = 0
fo = open(sortFileList)
run = 1
while (run):
reg_start = time.time()
dbmKey = fo.readline()
if (dbmKey.strip() == ""):
run = 0
continue
fileInfo = fileListDbm.get(dbmKey[0:-1])
filename = fileInfo[0]
diskId = fileInfo[1]
mimeType = fileInfo[2]
# Register the file. Check first, that exactly this file is
# not already registered. In case it is, the file will be rejected.
regPi = srvObj.getCfg().register_plugins[mimeType]
logger.debug("Plugin found for %s: %s", mimeType, regPi)
params = ngamsPlugInApi.parseRawPlugInPars(regPi.pars)
tmpReqPropsObj = ngamsReqProps.ngamsReqProps().\
setMimeType(mimeType).\
setStagingFilename(filename).\
setTargDiskInfo(diskInfoDic[diskId]).\
setHttpMethod(NGAMS_HTTP_GET).\
setCmd(NGAMS_REGISTER_CMD).\
setSize(os.path.getsize(filename)).\
setFileUri(filename).\
setNoReplication(1)
tmpFileObj = ngamsFileInfo.ngamsFileInfo()
try:
# Invoke Registration Plug-In.
piName = regPi.name
plugInMethod = loadPlugInEntryPoint(piName)
piRes = plugInMethod(srvObj, tmpReqPropsObj, params)
del tmpReqPropsObj
# Check if this file is already registered on this disk. In case
# yes, it is not registered again.
files = srvObj.db.getFileSummary1(srvObj.getHostId(),
[piRes.getDiskId()],
[piRes.getFileId()])
fileRegistered = 0
for tmpFileInfo in files:
tmpMtPt = tmpFileInfo[ngamsDbCore.SUM1_MT_PT]
tmpFilename = tmpFileInfo[ngamsDbCore.SUM1_FILENAME]
tmpComplFilename = os.path.normpath(tmpMtPt + "/" +\
tmpFilename)
if (tmpComplFilename == filename):
fileRegistered = 1
break
if (fileRegistered):
fileRejectCount += 1
tmpMsgForm = "REJECTED: File with File ID/Version: %s/%d " +\
"and path: %s is already registered on disk " +\
"with Disk ID: %s"
tmpMsg = tmpMsgForm % (piRes.getFileId(),
piRes.getFileVersion(), filename,
piRes.getDiskId())
logger.warning(tmpMsg + ". File is not registered again.")
if (emailNotif):
tmpFileObj.\
setDiskId(diskId).setFilename(filename).\
setTag(tmpMsg)
regDbm.addIncKey(tmpFileObj)
if (reqPropsObj):
reqPropsObj.incActualCount(1)
ngamsHighLevelLib.stdReqTimeStatUpdate(
srvObj, reqPropsObj, regTimeAccu)
regTimeAccu += time.time() - reg_start
fileCount += 1
continue
# Calculate checksum. We maintain the old name for backwards
# compatibility
crc_variant = srvObj.cfg.getCRCVariant()
if crc_variant == ngamsFileUtils.CHECKSUM_CRC32_INCONSISTENT:
crc_variant = 'ngamsGenCrc32'
checksum = ngamsFileUtils.get_checksum(65536, filename,
crc_variant) or ''
# Move file and update information about file in the NGAS DB.
mvFile(filename, piRes.getCompleteFilename())
ngamsArchiveUtils.updateFileInfoDb(srvObj, piRes, checksum,
crc_variant)
ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), piRes)
ngamsLib.makeFileReadOnly(piRes.getCompleteFilename())
if (emailNotif):
uncomprSize = piRes.getUncomprSize()
ingestDate = time.time()
creDateSecs = getFileCreationTime(filename)
tmpFileObj.\
setDiskId(diskId).\
setFilename(filename).\
setFileId(piRes.getFileId()).\
setFileVersion(piRes.getFileVersion()).\
setFormat(piRes.getFormat()).\
setFileSize(piRes.getFileSize()).\
setUncompressedFileSize(uncomprSize).\
setCompression(piRes.getCompression()).\
setIngestionDate(ingestDate).\
setIgnore(0).\
setChecksum(checksum).\
setChecksumPlugIn(crc_variant).\
setFileStatus(NGAMS_FILE_STATUS_OK).\
setCreationDate(creDateSecs).\
setTag("REGISTERED")
fileRegCount += 1
# If running as a cache archive, update the Cache New Files DBM
# with the information about the new file.
if (srvObj.getCachingActive()):
fileVer = fio.getFileVersion()
ngamsCacheControlThread.addEntryNewFilesDbm(
srvObj, diskId, piRes.getFileId(), fileVer, filename)
# Generate a confirmation log entry.
msg = genLog("NGAMS_INFO_FILE_REGISTERED", [
filename,
piRes.getFileId(),
piRes.getFileVersion(),
piRes.getFormat()
])
time.sleep(0.005)
regTime = time.time() - reg_start
msg = msg + ". Time: %.3fs." % (regTime)
logger.info(msg, extra={'to_syslog': 1})
except Exception as e:
errMsg = genLog("NGAMS_ER_FILE_REG_FAILED", [filename, str(e)])
logger.error(errMsg)
if (emailNotif):
tmpFileObj.\
setDiskId(diskId).setFilename(filename).\
setTag(errMsg)
fileFailCount += 1
regTime = time.time() - reg_start
# TODO (rtobar, 2016-01): Why don't we raise an exception here?
# Otherwise the command appears as successful on the
# client-side
# Add the file information in the registration report.
if (emailNotif): regDbm.addIncKey(tmpFileObj)
# Update request status time information.
regTimeAccu += regTime
if (reqPropsObj):
reqPropsObj.incActualCount(1)
ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj,
regTimeAccu)
fileCount += 1
fo.close()
rmFile(sortFileList)
if (emailNotif): regDbm.sync()
del fileListDbm
rmFile(fileListDbmName + "*")
# Final update of the Request Status.
if (reqPropsObj):
if (reqPropsObj.getExpectedCount() and reqPropsObj.getActualCount()):
complPercent = (100.0 * (float(reqPropsObj.getActualCount()) /
float(reqPropsObj.getExpectedCount())))
else:
complPercent = 100.0
reqPropsObj.setCompletionPercent(complPercent, 1)
reqPropsObj.setCompletionTime(1)
srvObj.updateRequestDb(reqPropsObj)
# Send Register Report with list of files cloned to a possible
# requestor(select) of this.
if (emailNotif):
xmlStat = 0
if (xmlStat):
# Create an instance of the File List Class, used to store the
# Registration Report.
regStat = ngamsFileList.\
ngamsFileList("FILE_REGISTRATION_STATUS",
"Status report for file " +\
"registration")
# Loop over the file objects in the BSD DB and add these
# in the status object.
regDbm.initKeyPtr()
while (1):
key, tmpFileObj = regDbm.getNext()
if (not key): break
regStat.addFileInfoObj(tmpFileObj)
# Set overall status of registration procedure.
regStat.setStatus("Files Found: " + str(fileCount + 1) + ", "+\
"Files Registered: " + str(fileRegCount) +\
", " +\
"Files Failed: " + str(fileFailCount) + ", " +\
"Files Rejected: " + str(fileRejectCount))
status = srvObj.genStatus(NGAMS_SUCCESS,
"REGISTER command status report").\
addFileList(regStat)
statRep = status.genXmlDoc()
statRep = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, statRep)
mimeType = NGAMS_XML_MT
else:
# Generate a 'simple' ASCII report.
statRep = tmpFilePat + "_NOTIF_EMAIL.txt"
fo = open(statRep, "w")
if (reqPropsObj):
path = reqPropsObj.getHttpPar("path")
else:
path = "-----"
if (fileCount):
timePerFile = (regTimeAccu / fileCount)
else:
timePerFile = 0
tmpFormat = "REGISTER STATUS REPORT:\n\n" +\
"==Summary:\n\n" +\
"Date: %s\n" +\
"NGAS Host: %s\n" +\
"Search Path: %s\n" +\
"Total Number of Files: %d\n" +\
"Number of Registered Files: %d\n" +\
"Number of Failed Files: %d\n" +\
"Number of Rejected Files: %d\n" +\
"Total processing time (s): %.3f\n" +\
"Handling time per file (s): %.3f\n\n" +\
"==File List:\n\n"
fo.write(tmpFormat % (toiso8601(), srvObj.getHostId(), path,
fileCount, fileRegCount, fileFailCount,
fileRejectCount, regTimeAccu, timePerFile))
tmpFormat = "%-80s %-32s %-3s %-10s\n"
fo.write(tmpFormat %\
("Filename", "File ID", "Ver", "Status"))
fo.write(tmpFormat % (80 * "-", 32 * "-", 3 * "-", 10 * "-"))
regDbm.initKeyPtr()
while (1):
key, tmpFileObj = regDbm.getNext()
if (not key): break
mtPt = diskInfoDic[tmpFileObj.getDiskId()].getMountPoint()
filename = os.path.normpath(mtPt + "/" +\
tmpFileObj.getFilename())
line = tmpFormat %\
(filename, tmpFileObj.getFileId(),
str(tmpFileObj.getFileVersion()),
tmpFileObj.getTag())
fo.write(line)
fo.write(128 * "-")
fo.write("\n\n==END\n")
fo.close()
mimeType = NGAMS_TEXT_MT
# Send out the status report.
emailAdrList = reqPropsObj.getHttpPar("notif_email").split(",")
attachmentName = "RegisterStatusReport"
if (reqPropsObj.hasHttpPar("path")):
attachmentName += "-" + reqPropsObj.getHttpPar("path").\
replace("/", "_")
ngamsNotification.notify(srvObj.host_id,
srvObj.cfg,
NGAMS_NOTIF_INFO,
"REGISTER STATUS REPORT",
statRep,
recList=emailAdrList,
force=1,
contentType=mimeType,
attachmentName=attachmentName)
del regDbm
rmFile(regDbmName + "*")
rmFile(statRep)
# Generate final status log + exit.
if (fileCount > 0):
timePerFile = (regTimeAccu / fileCount)
else:
timePerFile = 0.0
msg = "Registration procedure finished processing Register Request - " + \
"terminating. Files handled: %d. Total time: %.3fs. " + \
"Average time per file: %.3fs."
logger.debug(msg, fileCount, regTimeAccu, timePerFile)
def ngamsGLEAM_RRI_FilterPI(srvObj,
plugInPars,
filename,
fileId,
fileVersion=-1,
reqPropsObj=None):
"""
srvObj: Reference to NG/AMS Server Object (ngamsServer).
plugInPars: Parameters to take into account for the plug-in
execution (string).
fileId: File ID for file to test (string).
filename: Filename of (complete) (string).
fileVersion: Version of file to test (integer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: 0 if the file does not match, 1 if it matches the
conditions (integer/0|1).
"""
match = 0
fn, fext = os.path.splitext(fileId)
if (fext.lower() in file_ext
and # only send FITS files, no measurement sets
_shouldSend(fileId)
and # # only send files satisfying certain string pattern criteria
srvObj.getDb().isLastVersion(
fileId, fileVersion)): # only send the (known) latest version
parDic = []
pars = ""
if ((plugInPars != "") and (plugInPars != None)):
pars = plugInPars
elif (reqPropsObj != None):
if (reqPropsObj.hasHttpPar("plug_in_pars")):
pars = reqPropsObj.getHttpPar("plug_in_pars")
parDic = ngamsPlugInApi.parseRawPlugInPars(pars)
if (not parDic.has_key("remote_host")
or not parDic.has_key("remote_port")):
errMsg = "ngamsGLEAM_VUW_FilterPI: Missing Plug-In Parameter: " +\
"remote_host / remote_port"
#raise Exception, errMsg
logger.error(errMsg)
return 1 # matched as if the remote checking is done
host = parDic["remote_host"]
sport = parDic["remote_port"]
if (not sport.isdigit()):
errMsg = "ngamsGLEAM_VUW_FilterPI: Invalid port number: " + sport
logger.error(errMsg)
return 1 # matched as if the filter does not exist
port = int(sport)
# Perform the matching.
client = ngamsPClient.ngamsPClient(host,
port,
timeOut=NGAMS_SOCK_TIMEOUT_DEF)
try:
if (fileVersion == -1):
fileVersion = 1
rest = client.get_status(NGAMS_STATUS_CMD,
pars=[["file_id", fileId],
["file_version", fileVersion]])
# since the queue will be sorted based on ingestion date, this will ensure the versions are sent by order:
# e.g. version1, version2, version3, otherwise, this method will have disordered versions sent
if (rest.getStatus().find(NGAMS_FAILURE) != -1):
return 1 # matched since file id does not exist
except Exception as e:
errMsg = "Error occurred during checking remote file status " +\
"ngamsGLEAM_VUW_FilterPI. Exception: " + str(e)
logger.error(errMsg)
return 1 # matched as if the filter does not exist
#info(5, "filter return status = " + rest.getStatus())
#info(4, "filter match = " + str(match))
return match
def _registerExec(srvObj,
fileListDbmName,
tmpFilePat,
diskInfoDic,
reqPropsObj = None):
"""
Register the files listed in the File List DBM (ngamsDbm), which
match the mime-type(s) either specified in the 'mimeType' parameter,
or if this is not specified, which match all the mime-types specified
in the configuration file.
When the registration procedure has been executed, the function sends
an Email Notification message indicating which files were registered
if the HTTP parameter 'notif_email' is given.
The functions creates a File Info Objects per file handled and
writes this in a temporary file, which is a DBM file. The
keys in this DB is simply the file number in the sequence of files
handled, pointing to a pickled ngamsFileInfo object.
Each of the File Info Objects indicates if the file was registered or not.
This is done by setting the tag of the File Info Object to one of the
following values:
REGISTERED: The file was successfully registered in the NGAS DB
FAILED: The file was selected for registration but could not
be properly registered because of inconsistencies.
The status will be of the format: 'FAILED[: <reason>]'.
REJECTED: A file found under the specified path directory was
not accepted for cloning, usually because the mime-type
was not correct. The status will be of the format:
'REJECTED[: <reason>]'.
Note, that registration is comparable to archiving of files. For that
reason a DAPI must be provided for each type of file that should be
registered. If this is not fullfilled, the file registration will
fail.
Only files stored on one of the NGAS disks configured in the
configuration file, are considered. Files stored in other locations
are ignored.
srvObj: Instance of NG/AMS Server object (ngamsServer).
fileListDbmName: Name of a DBM containing the information
about the files to be registered. Each element in the
list is referred to by a key, which is a number.
These points to a pickled list for each file containing
the following information:
[<Filename>, <Disk ID>, <Mime-Type>]
The information for each disk concerned is also contained
in the DB referred to by its Disk ID and by its mount
point. The data is a pickled instance of the ngamsDiskInfo
class (string).
tmpFilePat: Pattern for temporary files used during the registration
process (string).
diskInfoDic: Dictionary with Disk IDs as keys pointing to the info
about the disk (dictionary/ngamsDiskInfo).
reqPropsObj: If an NG/AMS Request Properties Object is given, the
Request Status will be updated as the request is carried
out (ngamsReqProps).
Returns: Void.
"""
T = TRACE()
emailNotif = 0
if (reqPropsObj):
if (reqPropsObj.hasHttpPar("notif_email")):
emailNotif = 1
# Create the temporary BSD DB to contain the information for the
# Email Notification Message.
if (emailNotif):
regDbmName = tmpFilePat + "_NOTIF_EMAIL"
regDbm = ngamsDbm.ngamsDbm(regDbmName, writePerm = 1)
# Open the DBM containing the list of files to (possibly) register.
fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm = 1)
# Want to parse files in alphabetical order.
# TODO: Portatibility issue. Try to avoid UNIX shell commands for sorting.
tmpFileList = tmpFilePat + "_FILE_LIST"
rmFile(tmpFileList)
fo = open(tmpFileList, "w")
fileListDbm.initKeyPtr()
while (1):
dbmKey, fileInfo = fileListDbm.getNext()
if (not dbmKey): break
fo.write(dbmKey + "\n")
fo.close()
sortFileList = tmpFilePat + "_SORT_FILE_LIST"
rmFile(sortFileList)
shellCmd = "sort %s > %s" % (tmpFileList, sortFileList)
stat, out = commands.getstatusoutput(shellCmd)
if (stat != 0):
raise Exception, "Error executing command: %s. Error: %s" %\
(shellCmd, str(out))
rmFile(tmpFileList)
# Go through each file in the list, check if the mime-type is among the
# ones, which apply for registration. If yes try to register the file
# by invoking the corresponding DAPI on the file.
checksumPlugIn = srvObj.getCfg().getChecksumPlugIn().strip()
fileRegCount = 0
fileFailCount = 0
fileRejectCount = 0
regTimeAccu = 0.0
fileCount = 0
fo = open(sortFileList)
run = 1
while (run):
reg_start = time.time()
dbmKey = fo.readline()
if (dbmKey.strip() == ""):
run = 0
continue
fileInfo = fileListDbm.get(dbmKey[0:-1])
filename = fileInfo[0]
diskId = fileInfo[1]
mimeType = fileInfo[2]
# Register the file. Check first, that exactly this file is
# not already registered. In case it is, the file will be rejected.
regPi = srvObj.getCfg().register_plugins[mimeType]
logger.debug("Plugin found for %s: %s", mimeType, regPi)
params = ngamsPlugInApi.parseRawPlugInPars(regPi.pars)
tmpReqPropsObj = ngamsReqProps.ngamsReqProps().\
setMimeType(mimeType).\
setStagingFilename(filename).\
setTargDiskInfo(diskInfoDic[diskId]).\
setHttpMethod(NGAMS_HTTP_GET).\
setCmd(NGAMS_REGISTER_CMD).\
setSize(os.path.getsize(filename)).\
setFileUri(filename).\
setNoReplication(1)
tmpFileObj = ngamsFileInfo.ngamsFileInfo()
try:
# Invoke Registration Plug-In.
piName = regPi.name
plugInMethod = loadPlugInEntryPoint(piName)
piRes = plugInMethod(srvObj, tmpReqPropsObj, params)
del tmpReqPropsObj
# Check if this file is already registered on this disk. In case
# yes, it is not registered again.
files = srvObj.db.getFileSummary1(srvObj.getHostId(), [piRes.getDiskId()],
[piRes.getFileId()])
fileRegistered = 0
for tmpFileInfo in files:
tmpMtPt = tmpFileInfo[ngamsDbCore.SUM1_MT_PT]
tmpFilename = tmpFileInfo[ngamsDbCore.SUM1_FILENAME]
tmpComplFilename = os.path.normpath(tmpMtPt + "/" +\
tmpFilename)
if (tmpComplFilename == filename):
fileRegistered = 1
break
if (fileRegistered):
fileRejectCount += 1
tmpMsgForm = "REJECTED: File with File ID/Version: %s/%d " +\
"and path: %s is already registered on disk " +\
"with Disk ID: %s"
tmpMsg = tmpMsgForm % (piRes.getFileId(),
piRes.getFileVersion(), filename,
piRes.getDiskId())
logger.warning(tmpMsg + ". File is not registered again.")
if (emailNotif):
tmpFileObj.\
setDiskId(diskId).setFilename(filename).\
setTag(tmpMsg)
regDbm.addIncKey(tmpFileObj)
if (reqPropsObj):
reqPropsObj.incActualCount(1)
ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj,
regTimeAccu)
regTimeAccu += time.time() - reg_start
fileCount += 1
continue
# Calculate checksum (if plug-in specified).
if (checksumPlugIn != ""):
logger.debug("Invoking Checksum Plug-In: %s to handle file: %s",
checksumPlugIn, filename)
plugInMethod = loadPlugInEntryPoint(checksumPlugIn)
checksum = plugInMethod(srvObj, filename, 0)
else:
checksum = ""
# Move file and update information about file in the NGAS DB.
mvFile(filename, piRes.getCompleteFilename())
ngamsArchiveUtils.updateFileInfoDb(srvObj, piRes, checksum,
checksumPlugIn)
ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), piRes)
ngamsLib.makeFileReadOnly(piRes.getCompleteFilename())
if (emailNotif):
uncomprSize = piRes.getUncomprSize()
ingestDate = time.time()
creDateSecs = getFileCreationTime(filename)
tmpFileObj.\
setDiskId(diskId).\
setFilename(filename).\
setFileId(piRes.getFileId()).\
setFileVersion(piRes.getFileVersion()).\
setFormat(piRes.getFormat()).\
setFileSize(piRes.getFileSize()).\
setUncompressedFileSize(uncomprSize).\
setCompression(piRes.getCompression()).\
setIngestionDate(ingestDate).\
setIgnore(0).\
setChecksum(checksum).\
setChecksumPlugIn(checksumPlugIn).\
setFileStatus(NGAMS_FILE_STATUS_OK).\
setCreationDate(creDateSecs).\
setTag("REGISTERED")
fileRegCount += 1
# If running as a cache archive, update the Cache New Files DBM
# with the information about the new file.
if (srvObj.getCachingActive()):
fileVer = fio.getFileVersion()
ngamsCacheControlThread.addEntryNewFilesDbm(srvObj, diskId,
piRes.getFileId(),
fileVer, filename)
# Generate a confirmation log entry.
msg = genLog("NGAMS_INFO_FILE_REGISTERED",
[filename, piRes.getFileId(), piRes.getFileVersion(),
piRes.getFormat()])
time.sleep(0.005)
regTime = time.time() - reg_start
msg = msg + ". Time: %.3fs." % (regTime)
logger.info(msg, extra={'to_syslog': 1})
except Exception, e:
errMsg = genLog("NGAMS_ER_FILE_REG_FAILED", [filename, str(e)])
logger.error(errMsg)
if (emailNotif):
tmpFileObj.\
setDiskId(diskId).setFilename(filename).\
setTag(errMsg)
fileFailCount += 1
regTime = time.time() - reg_start
# TODO (rtobar, 2016-01): Why don't we raise an exception here?
# Otherwise the command appears as successful on the
# client-side
# Add the file information in the registration report.
if (emailNotif): regDbm.addIncKey(tmpFileObj)
# Update request status time information.
regTimeAccu += regTime
if (reqPropsObj):
reqPropsObj.incActualCount(1)
ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj,
regTimeAccu)
fileCount += 1
def ngamsMWA_Compress_JobPlugin(srvObj, plugInPars, filename, fileId,
fileVersion, diskId):
"""
srvObj: Reference to NG/AMS Server Object (ngamsServer).
plugInPars: Parameters to take into account for the plug-in
execution (string).(e.g. scale_factor=4,threshold=1E-5)
fileId: File ID for file to test (string).
filename: Filename of (complete) (string).
fileVersion: Version of file to test (integer).
Returns: the return code of the compression plugin (integer).
"""
pars = ""
sf = 4 # scaling factor
th = 1E-5 # threshold
bins = 0
remove_uc = 0
if ((plugInPars != "") and (plugInPars != None)):
pars = plugInPars
parDic = ngamsPlugInApi.parseRawPlugInPars(pars)
if (parDic.has_key('scale_factor')):
sf = int(parDic['scale_factor'])
if (parDic.has_key('threshold')):
th = float(parDic['threshold'])
if (parDic.has_key('bins')):
bins = int(parDic['bins'])
if (parDic.has_key('remove_uc')):
remove_uc = int(parDic['remove_uc'])
if (bins):
binstr = '-h %d' % bins
else:
binstr = ''
newfn = '%s/%s' % (work_dir, os.path.basename(filename))
archive_host = get_contact_ip(
srvObj.getCfg()
) # archive host must be on the same machine as the data mover or job runner
cmd = "%s -d %d %s %s %s" % (uvcompress, sf, binstr, filename, newfn)
cmd1 = "%s -host %s -port 7777 -fileUri %s -cmd QARCHIVE -mimeType application/octet-stream " % (
archive_client, archive_host, newfn)
cmd2 = "curl http://%s:7777/DISCARD?file_id=%s\\&file_version=%d\\&disk_id=%s\\&execute=1" % (
archive_host, fileId, fileVersion, diskId)
cmd3 = "rm %s" % newfn
if (debug):
logger.debug('*******************************************')
logger.debug(cmd)
logger.debug(cmd1)
if (remove_uc):
logger.debug(cmd2)
logger.debug(cmd3)
logger.debug('*******************************************')
return (0, 'Compressed OK')
else:
re = commands.getstatusoutput(cmd)
if (0 == re[0]):
if (not bins):
retstr = re[1].split('\n')[-1] # just get the elapsed time
else:
retstr = re[1].split('------ Histogram ------\n')[1]
# archive it back
# TODO - enable time out!!
re = commands.getstatusoutput(cmd1)
if (0 == re[0]):
logger.debug('Successfully re-archived the compressed file %s',
newfn)
else:
logger.error('Fail to re-archive compressed file %s: %s',
newfn, re[1])
return (re[0], re[1])
if (remove_uc):
# remove the uncompressed file if necessary
re = commands.getstatusoutput(cmd2)
if (0 == re[0]):
logger.debug(
'Successfully DISCARDED the uncompressed file %s',
filename)
else:
logger.warning('Fail to DISCARD the uncompressed file %s',
filename)
# remove the temp file
re = commands.getstatusoutput(cmd3)
if (0 != re[0]):
logger.warning('Fail to remove the temp compressed file %s',
newfn)
return (0, retstr)
else:
logger.error('Fail to compress file %s: %s', filename, re[1])
return (re[0], re[1])
def ngamsMWA_MIT_NoTape_FilterPI(srvObj,
plugInPars,
filename,
fileId,
fileVersion=-1,
reqPropsObj=None):
"""
srvObj: Reference to NG/AMS Server Object (ngamsServer).
plugInPars: Parameters to take into account for the plug-in
execution (string).
fileId: File ID for file to test (string).
filename: Filename of (complete) (string).
fileVersion: Version of file to test (integer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: 0 if the file does not match, 1 if it matches the
conditions (integer/0|1).
"""
match = 0
projectId = ''
try:
projectId = pyfits.getval(filename, 'PROJID')
except:
err = "Did not find keyword PROJID in FITS file or PROJID illegal"
errMsg = genLog(
"NGAMS_ER_DAPI_BAD_FILE",
[os.path.basename(filename), "ngamsMWA_MIT_FilterPlugIn", err])
#raise Exception, errMsg
#so still possible to deliver if the file is not there yet
"""
if (projectId == "'C105'" or projectId == "'C106'"):
return 0
"""
# Parse plug-in parameters.
pars = ""
if ((plugInPars != "") and (plugInPars != None)):
pars = plugInPars
elif (reqPropsObj != None):
if (reqPropsObj.hasHttpPar("plug_in_pars")):
pars = reqPropsObj.getHttpPar("plug_in_pars")
parDic = ngamsPlugInApi.parseRawPlugInPars(pars)
if (not parDic.has_key("remote_host") or not parDic.has_key("remote_port")
or not parDic.has_key("project_id")):
errMsg = "ngamsMWACheckRemoteFilterPlugin: Missing Plug-In Parameter: " +\
"remote_host / remote_port / project_id"
#raise Exception, errMsg
logger.error(errMsg)
return 1 # matched as if the filter did not exist
host = parDic["remote_host"]
sport = parDic["remote_port"]
proj_ids = parDic["project_id"]
if (proj_ids and len(proj_ids)):
for proj_id in proj_ids.split(proj_separator):
eor_list.append("'%s'" % proj_id)
if (not (projectId in eor_list)):
logger.debug('File %s is not EOR project', fileId)
return 0
if (not sport.isdigit()):
errMsg = "ngamsMWACheckRemoteFilterPlugin: Invalid port number: " + sport
logger.error(errMsg)
return 1 # matched as if the filter does not exist
port = int(sport)
# Perform the matching.
client = ngamsPClient.ngamsPClient(host,
port,
timeOut=NGAMS_SOCK_TIMEOUT_DEF)
try:
rest = client.get_status(NGAMS_STATUS_CMD, pars=[["file_id", fileId]])
except Exception:
errMsg = "Error occurred during checking remote file status " +\
"ngamsMWACheckRemoteFilterPlugin"
logger.exception(errMsg)
return 1 # matched as if the filter does not exist
#info(5, "filter return status = " + rest.getStatus())
if (rest.getStatus().find(NGAMS_FAILURE) != -1):
tname = threading.current_thread().name
beingSent = srvObj._subscrDeliveryFileDic.values()
for fi in beingSent:
if (srvObj._subscrDeliveryFileDic[tname] != fi and fi[0] == fileId
and fi[2] == fileVersion):
return 0 # this file is currently being sent, so do not send it again
match = 1
#info(4, "filter match = " + str(match))
return match
def ngamsMWACheckRemoteFilterPlugin(srvObj,
plugInPars,
filename,
fileId,
fileVersion=-1,
reqPropsObj=None):
"""
Example Filter Plug-In used to filter on a given mime-type. In case the
file referenced has the mime-type as specified in the plug-in parameters,
the file being tested is selected.
srvObj: Reference to NG/AMS Server Object (ngamsServer).
plugInPars: Parameters to take into account for the plug-in
execution (string).
fileId: File ID for file to test (string).
filename: Filename of (complete) (string).
fileVersion: Version of file to test (integer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: 0 if the file does not match, 1 if it matches the
conditions (integer/0|1).
"""
match = 0
# Parse plug-in parameters.
parDic = []
pars = ""
if ((plugInPars != "") and (plugInPars != None)):
pars = plugInPars
elif (reqPropsObj != None):
if (reqPropsObj.hasHttpPar("plug_in_pars")):
pars = reqPropsObj.getHttpPar("plug_in_pars")
parDic = ngamsPlugInApi.parseRawPlugInPars(pars)
if (not parDic.has_key("remote_host")
or not parDic.has_key("remote_port")):
errMsg = "ngamsMWACheckRemoteFilterPlugin: Missing Plug-In Parameter: " +\
"remote_host and/or remote_port"
#raise Exception, errMsg
logger.error(errMsg)
return 1 # matched as if the filter does not exist
host = parDic["remote_host"]
sport = parDic["remote_port"]
if (not sport.isdigit()):
errMsg = "ngamsMWACheckRemoteFilterPlugin: Invalid port number: " + sport
logger.error(errMsg)
return 1 # matched as if the filter does not exist
port = int(sport)
# Perform the matching.
client = ngamsPClient.ngamsPClient(host, port)
try:
rest = client.get_status(NGAMS_STATUS_CMD, pars=[["file_id", fileId]])
except Exception:
errMsg = "Error occurred during checking remote file status " +\
"ngamsMWACheckRemoteFilterPlugin"
logger.exception(errMsg)
return 1 # matched as if the filter does not exist
logger.debug("filter return status = %s", rest.getStatus())
if (rest.getStatus().find(NGAMS_FAILURE) != -1):
match = 1
logger.debug("filter match = %s", str(match))
return match
def ngamsMWA_MIT_FilterPlugin(srvObj,
plugInPars,
filename,
fileId,
fileVersion=-1,
reqPropsObj=None):
"""
srvObj: Reference to NG/AMS Server Object (ngamsServer).
plugInPars: Parameters to take into account for the plug-in
execution (string).
fileId: File ID for file to test (string).
filename: Filename of (complete) (string).
fileVersion: Version of file to test (integer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: 0 if the file does not match, 1 if it matches the
conditions (integer/0|1).
"""
pars = ""
if ((plugInPars != "") and (plugInPars != None)):
pars = plugInPars
elif (reqPropsObj != None):
if (reqPropsObj.hasHttpPar("plug_in_pars")):
pars = reqPropsObj.getHttpPar("plug_in_pars")
parDic = ngamsPlugInApi.parseRawPlugInPars(pars)
if (not parDic.has_key("remote_host") or not parDic.has_key("remote_port")
or not parDic.has_key("project_id")):
errMsg = "ngamsMWACheckRemoteFilterPlugin: Missing Plug-In Parameter: " +\
"remote_host / remote_port / project_id"
logger.error(errMsg)
return 0
proj_ids = parDic["project_id"]
if (not (proj_ids and len(proj_ids))):
return 0
for proj_id in proj_ids.split(proj_separator):
eor_list.append("'%s'" % proj_id)
fspi = srvObj.getCfg().getFileStagingPlugIn()
if (not fspi):
offline = -1
else:
logger.info("Invoking FSPI.isFileOffline: %s to check file: %s", fspi,
filename)
isFileOffline = loadPlugInEntryPoint(fspi, 'isFileOffline')
offline = isFileOffline(filename)
try:
if (offline == 1 or offline == -1):
# if the file is on Tape or query error, query db instead, otherwise implicit tape staging will block all other threads!!
logger.debug('File %s appears on Tape, connect to MWA DB to check',
filename)
projId = getProjectIdFromMWADB(fileId)
if (not projId or projId == ''):
logger.error('Cannot get project id from MWA DB for file %s',
fileId)
return 0
projectId = "'%s'" % projId # add single quote to be consistent with FITS header keywords
else:
projectId = pyfits.getval(filename, 'PROJID')
except:
err = "Did not find keyword PROJID in FITS file or PROJID illegal"
errMsg = genLog(
"NGAMS_ER_DAPI_BAD_FILE",
[os.path.basename(filename), "ngamsMWA_MIT_FilterPlugIn", err])
return 0
if (projectId in eor_list):
logger.debug('File %s added', fileId)
return 1
else:
return 0
def ngamsMWA_MIT_Priority_FilterPI(srvObj,
plugInPars,
filename,
fileId,
fileVersion=-1,
reqPropsObj=None):
"""
srvObj: Reference to NG/AMS Server Object (ngamsServer).
plugInPars: Parameters to take into account for the plug-in
execution (string).
fileId: File ID for file to test (string).
filename: Filename of (complete) (string).
fileVersion: Version of file to test (integer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: 0 if the file does not match, 1 if it matches the
conditions (integer/0|1).
"""
match = 0
obsId = fileId.split('_')[0]
if (not obsId in obs_list):
return 0
# Parse plug-in parameters.
pars = ""
if ((plugInPars != "") and (plugInPars != None)):
pars = plugInPars
elif (reqPropsObj != None):
if (reqPropsObj.hasHttpPar("plug_in_pars")):
pars = reqPropsObj.getHttpPar("plug_in_pars")
parDic = ngamsPlugInApi.parseRawPlugInPars(pars)
if (not parDic.has_key("remote_host")
or not parDic.has_key("remote_port")):
errMsg = "ngamsMWACheckRemoteFilterPlugin: Missing Plug-In Parameter: " +\
"remote_host / remote_port"
#raise Exception, errMsg
logger.error(errMsg)
return 1 # matched as if the filter did not exist
host = parDic["remote_host"]
sport = parDic["remote_port"]
if (not sport.isdigit()):
errMsg = "ngamsMWACheckRemoteFilterPlugin: Invalid port number: " + sport
logger.error(errMsg)
return 1 # matched as if the filter does not exist
port = int(sport)
# Perform the matching.
client = ngamsPClient.ngamsPClient(host,
port,
timeOut=NGAMS_SOCK_TIMEOUT_DEF)
try:
rest = client.get_status(NGAMS_STATUS_CMD, pars=[["file_id", fileId]])
except Exception:
errMsg = "Error occurred during checking remote file status " +\
"ngamsMWACheckRemoteFilterPlugin"
logger.exception(errMsg)
return 1 # matched as if the filter does not exist
#info(5, "filter return status = " + rest.getStatus())
if (rest.getStatus().find(NGAMS_FAILURE) != -1):
match = 1
#info(4, "filter match = " + str(match))
return match
def ngamsGLEAM_Decompress_JobPlugin(srvObj, plugInPars, filename, fileId,
fileVersion, diskId):
"""
srvObj: Reference to NG/AMS Server Object (ngamsServer).
plugInPars: Parameters to take into account for the plug-in
execution (string).(e.g. scale_factor=4,threshold=1E-5)
fileId: File ID for file to test (string).
filename: Filename of (complete) (string).
fileVersion: Version of file to test (integer).
Returns: the return code of the compression plugin (integer).
"""
pars = ""
remove_uc = 0
timeout = 600 # each command should not run more than 10 min, otherwise something is wrong
if ((plugInPars != "") and (plugInPars != None)):
pars = plugInPars
parDic = ngamsPlugInApi.parseRawPlugInPars(pars)
if (parDic.has_key('remove_uc')):
remove_uc = int(parDic['remove_uc'])
if (parDic.has_key('timeout')):
timeout = int(parDic['timeout'])
if (timeout <= 0):
timeout = 600
archive_host = get_contact_ip(srvObj.getCfg())
#cmd1 = "%s -host %s -port 7777 -fileUri %s -cmd QARCHIVE -mimeType application/octet-stream " % (archive_client, archive_host, newfn)
cmd2 = "curl http://%s:7777/DISCARD?file_id=%s\\&file_version=%d\\&disk_id=%s\\&execute=1" % (
archive_host, fileId, fileVersion, diskId)
"""
if (debug):
info(3, '*******************************************')
info(3, cmd)
info(3, cmd1)
if (remove_uc):
info(3, cmd2)
info(3, cmd3)
info(3, '*******************************************')
return (0, 'Compressed OK')
else:
"""
#re = commands.getstatusoutput(cmd)
cmd = "tar xf %s -C %s" % (filename, work_dir)
logger.debug("Extracting %s to %s", filename, work_dir)
re = execCmd(cmd, timeout)
if (0 == re[0]):
# archive it back
obsId = fileId.split('_')[0]
imglist = glob('%s/%s/*.fits' % (work_dir, obsId))
errNo = 0
lasterrMsg = ''
for imgfile in imglist:
url = 'http://%s:7777/LARCHIVE?fileUri=%s\&mimeType=application/octet-stream\&file_version=%d\&no_versioning=1\&versioning=0' % (
archive_host, imgfile, fileVersion)
cmd1 = 'curl --connect-timeout %d %s' % (timeout, url)
logger.debug('Local archiving %s', cmd1)
re = commands.getstatusoutput(cmd1)
if (0 == re[0] and
(re[1].count('Successfully handled Archive Pull Request') >
0)):
logger.debug(
'Successfully re-archived the untarred FITS file %s',
imgfile)
else:
logger.error(
'Fail to re-archive the untarred FITS file %s: %s',
imgfile, re[1])
errNo += 1
lasterrMsg = re[1]
#return (re[0], re[1])
if (remove_uc and errNo == 0):
# remove the original file if necessary
re = execCmd(cmd2, timeout)
logger.debug('Removing the tar file %s', filename)
if (0 == re[0]):
logger.debug('Successfully DISCARDED the tar file %s',
filename)
else:
logger.warning('Fail to DISCARD the tar file %s', filename)
# remove the temp file
cmd3 = "rm -rf %s/%s" % (work_dir, obsId)
logger.debug("Removing the temp directory %s/%s", work_dir, obsId)
re = execCmd(cmd3, timeout)
if (0 != re[0]):
logger.warning('Fail to remove the temp untarred directory %s/%s',
work_dir, obsId)
if (errNo == 0):
return (0, 'Done')
else:
return (errNo, lasterrMsg.replace("'", ""))
else:
logger.error('Fail to untar file %s: %s', filename, re[1])
return (re[0], re[1])
def ngamsJob_MITDeliveryPlugin(srvObj,
plugInPars,
filename,
fileId,
fileVersion,
diskId):
"""
srvObj: Reference to NG/AMS Server Object (ngamsServer).
plugInPars: Parameters to take into account for the plug-in
execution (string).(e.g. scale_factor=4,threshold=1E-5)
fileId: File ID for file to test (string).
filename: Filename of (complete) (string).
fileVersion: Version of file to test (integer).
Returns: the return code of the compression plugin (integer).
"""
debug = 0
pars = ""
sf = 4 # scaling factor
th = 1E-5 # threshold
bins = 0
send_crc = 1
to_host = 'eor-10.mit.edu'
to_port = '7777'
if ((plugInPars != "") and (plugInPars != None)):
pars = plugInPars
parDic = ngamsPlugInApi.parseRawPlugInPars(pars)
if (parDic.has_key('scale_factor')):
sf = int(parDic['scale_factor'])
if (parDic.has_key('threshold')):
th = float(parDic['threshold'])
if (parDic.has_key('bins')):
bins = int(parDic['bins'])
if (parDic.has_key('send_crc')):
send_crc = int(parDic['send_crc'])
if (parDic.has_key('to_host')):
to_host = parDic['to_host']
if (parDic.has_key('to_port')):
to_port = parDic['to_port']
if (bins):
binstr = '-h %d' % bins
else:
binstr = ''
work_dir = srvObj.getCfg().getRootDirectory() + '/tmp/'
newfn = '%s/%s' % (work_dir, os.path.basename(filename))
cmd = "%s -d %d %s %s %s" % (uvcompress, sf, binstr, filename, newfn)
cmd3 = "rm %s" % newfn
if (os.path.exists(newfn)):
execCmd(cmd3)
mwaFits = isMWAVisFile(fileId)
sendRawFile = False
if (mwaFits):
re = execCmd(cmd)
#if (0 == re[0]):
if (os.WIFEXITED(re[0])): # if the child process exit normally (see http://linux.die.net/man/2/wait)
if (not bins):
retstr = re[1].split('\n')[-1] # just get the elapsed time
else:
retstr = re[1].split('------ Histogram ------\n')[1]
#bbcpurl = "curl --connect-timeout 7200 eor-12.mit.edu:7777/BBCPARC?fileUri=ngas%40146.118.84.67%3A/mnt/mwa01fs/MWA/testfs/output_320M_001.dat\&bport=7790\&bwinsize=%3D32m\&bnum_streams=12\&mimeType=application/octet-stream"
bbcpurl = "http://%s:%s/BBCPARC?fileUri=ngas%%40146.118.84.67%%3A%s&bport=7790&bwinsize=%%3D32m&bnum_streams=12&mimeType=application/octet-stream" % (to_host, to_port, newfn)
if (send_crc):
try:
crc32 = getFileCRC(newfn)
bbcpurl += "&bchecksum=%s" % str(crc32)
except Exception as crcexp:
logger.exception('Fail to calculate the file CRC %s', newfn)
return (500, str(crcexp).replace("'",""))
# send it thru bbcp
try:
resp = urllib2.urlopen(bbcpurl, timeout = 7200)
retstr = resp.read()
if (retstr.find("Successfully handled Archive") > -1):
logger.debug('Successfully compressed and sent the file %s', newfn)
else:
logger.error('Fail to sent the file %s: %s', newfn, retstr)
return (500, retstr.replace("'",""))
except Exception as exp:
warn_msg = ''
if (type(exp) is urllib2.HTTPError):
errors = exp.readlines()
for ee in errors:
warn_msg += ee
warn_msg = 'HTTP error: %s' % (warn_msg)
elif (type(exp) is urllib2.URLError):
warn_msg = 'Target NGAS server %s is down: %s' % (to_host, str(exp))
else:
warn_msg = 'Unexpected error: %s' % (str(exp))
logger.warning('Fail to sent the file %s: %s', newfn, warn_msg)
return (500, warn_msg.replace("'",""))
finally:
# remove the temp file THIS IS DANGEROUS!!
re = execCmd(cmd3) #be cautious when copying this line!!
if (0 != re[0]):
logger.warning('Fail to remove the temp compressed file %s', newfn)
return (0, retstr.replace("'",""))
else:
logger.error('Fail to compress file %s: %s', filename, re[1])
re = execCmd(cmd3)
if (0 != re[0]):
logger.warning('Fail to remove the temp compressed file %s', newfn)
sendRawFile = True
if ((not mwaFits) or sendRawFile):
#send uncompressed version
bbcpurl = "http://%s:%s/BBCPARC?fileUri=ngas%%40146.118.84.67%%3A%s&bport=7790&bwinsize=%%3D32m&bnum_streams=12&mimeType=application/octet-stream" % (to_host, to_port, filename)
if (send_crc):
try:
crc32 = getFileCRC(filename)
bbcpurl += "&bchecksum=%s" % str(crc32)
except Exception as crcexp:
logger.exception('Fail to calculate the file CRC %s', filename)
return (500, str(crcexp).replace("'",""))
# send it thru bbcp
try:
resp = urllib2.urlopen(bbcpurl, timeout = 7200)
retstr = resp.read()
if (retstr.find("Successfully handled Archive") > -1):
logger.debug('Successfully sent the uncompressed file %s', filename)
return (0, retstr.replace("'",""))
else:
logger.error('Fail to sent the file %s: %s', filename, retstr)
return (500, retstr.replace("'",""))
except Exception as exp:
warn_msg = ''
if (type(exp) is urllib2.HTTPError):
errors = exp.readlines()
for ee in errors:
warn_msg += ee
warn_msg = 'HTTP error: %s' % (warn_msg)
elif (type(exp) is urllib2.URLError):
warn_msg = 'Target NGAS server %s is down: %s' % (to_host, str(exp))
else:
warn_msg = 'Unexpected error: %s' % (str(exp))
logger.warning('Fail to sent the file %s: %s', filename, warn_msg)
return (500, warn_msg.replace("'",""))
def ngamsTestDppi1(srvObj, reqPropsObj, filename):
"""
This test DPPI extracts the main header from a FITS file requested from the
ESO Archive.
Depending on the the DPPI Plug-In Parameters it stores the result in a file
or in a buffer in memory.
This is made to work on small FITS files.
srvObj: Reference to instance of the NG/AMS Server
class (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
filename: Name of file to process (string).
Returns: DPPI return status object (ngamsDppiStatus).
"""
statusObj = ngamsDppiStatus.ngamsDppiStatus()
# Uncompress the file.
procFile, procDir = ngamsPlugInApi.prepProcFile(srvObj.getCfg(), filename)
subprocess.check_call(['gunzip', procFile])
procFile = procFile[0:procFile.rfind(".")]
# dpallot: fold fails miserably on Mac when dealing with binary files
#
# Process the output file.
#stat, out = commands.getstatusoutput("fold %s" % procFile)
#if stat:
# raise Exception('Problem while folding %s: %s' % (procFile, out))
head = []
with open(procFile, 'rb') as f:
while True:
line = f.read(80)
assert (line and len(line) == 80)
head.append(line)
head.append(b'\n')
if b'END' in line:
break
mimeType = "TEST-MIME-TYPE"
rawPiPars = srvObj.getCfg().dppi_plugins["ngamsTest.ngamsTestDppi1"].pars
cfgParDic = ngamsPlugInApi.parseRawPlugInPars(rawPiPars)
head.append(b"\n\nConfiguration Parameters:\n")
parList = list(cfgParDic)
parList.sort()
for par in parList:
head.append(six.b("%s=%s\n" % (par, cfgParDic[par])))
head.append(b"\nParameters Transferred:\n")
httpParsDic = reqPropsObj.getHttpParsDic()
httpPars = list(httpParsDic)
httpPars.sort()
for httpPar in httpPars:
head.append(six.b("%s=%s\n" % (httpPar, httpParsDic[httpPar])))
head.append(b"\nEOF\n")
buf = b''.join(head)
# Generate status.
if (cfgParDic["TARGET"] == "FILE"):
outFile = procFile + "_ngamsTestDppi1"
with open(outFile, "ab") as fo:
fo.write(buf)
resObj = ngamsDppiStatus.ngamsDppiResult(NGAMS_PROC_FILE, mimeType,
outFile, filename, procDir)
else:
resObj = ngamsDppiStatus.ngamsDppiResult(NGAMS_PROC_DATA, mimeType,
buf, filename)
statusObj.addResult(resObj)
return statusObj
def ngamsBrotherPT9200DxPlugIn(srvObj, label, reqPropsObj=None):
"""
Driver for printing labels on the label printer Brother PT-9200DX.
srvObj: Reference to instance of the NG/AMS Server
class (ngamsServer).
label: Label text to print (string).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: Void.
"""
plugInPars = srvObj.getCfg().getLabelPrinterPlugInPars()
logger.info("Executing plug-in ngamsBrotherPT9200DxPlugIn with parameters: " + \
"%s - Label: %s ...", plugInPars, label)
parDic = ngamsPlugInApi.parseRawPlugInPars(plugInPars)
# Get the font bit pattern dictionary.
fontDic = genFontsDictionary(parDic["font_file"])
# Generate the printer control code.
printerCode = fontDic["Header"]
for i in range(len(label)):
if (not fontDic.has_key(label[i])):
errMsg = "No font defintion for character: \"" + label[i] +\
"\" - in font definition file: " + parDic["font_file"] +\
" - cannot generate disk label: " + label
ngamsPlugInApi.notify(srvObj, NGAMS_NOTIF_ERROR,
"ngamsBrotherPT9200DxPlugIn: " +\
"ILLEGAL CHARACTER REQ. FOR PRINTING",
errMsg)
raise Exception(errMsg)
printerCode = printerCode + fontDic[label[i]]
printerCode = printerCode + fontDic["Trailer"]
# Generate printer file, write the printer control code.
tmpDir = ngamsPlugInApi.getTmpDir(srvObj.getCfg())
ngasId = srvObj.getHostId()
printerFilename = os.path.normpath(tmpDir + "/ngamsLabel_" + ngasId +
".prn")
fo = open(printerFilename, "w")
fo.write(printerCode)
fo.close()
# Write the printer code file to the device.
stat, out = ngamsPlugInApi.execCmd("cat " + printerFilename + " > " +\
parDic["dev"])
# This was previously excluded during the "test mode"; a proper way is by
# having a configurable parameter
if not 'keep_printer_file' in parDic:
os.system("rm -f " + printerFilename)
if (stat != 0):
errMsg = "Problem occurred printing label! Error: " + str(out)
ngamsPlugInApi.notify(srvObj, NGAMS_NOTIF_ERROR,
"ngamsBrotherPT9200DxPlugIn: " +\
"PROBLEM PRINTING LABEL", errMsg)
raise Exception(errMsg)
def ngamsGLEAM_VUW_FilterPI(srvObj,
plugInPars,
filename,
fileId,
fileVersion=-1,
reqPropsObj=None):
"""
srvObj: Reference to NG/AMS Server Object (ngamsServer).
plugInPars: Parameters to take into account for the plug-in
execution (string).
fileId: File ID for file to test (string).
filename: Filename of (complete) (string).
fileVersion: Version of file to test (integer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: 0 if the file does not match, 1 if it matches the
conditions (integer/0|1).
"""
match = 0
fn, fext = os.path.splitext(fileId)
if (
fext.lower() in file_ext
and srvObj.getDb().isLastVersion(fileId, fileVersion)
): # only send FITS files, no measurement sets, only send the (known) latest version
parDic = []
pars = ""
if ((plugInPars != "") and (plugInPars != None)):
pars = plugInPars
elif (reqPropsObj != None):
if (reqPropsObj.hasHttpPar("plug_in_pars")):
pars = reqPropsObj.getHttpPar("plug_in_pars")
parDic = ngamsPlugInApi.parseRawPlugInPars(pars)
if (not parDic.has_key("remote_host")
or not parDic.has_key("remote_port")):
errMsg = "ngamsGLEAM_VUW_FilterPI: Missing Plug-In Parameter: " +\
"remote_host / remote_port"
#raise Exception, errMsg
logger.error(errMsg)
return 1 # matched as if the remote checking is done
host = parDic["remote_host"]
sport = parDic["remote_port"]
if (not sport.isdigit()):
errMsg = "ngamsGLEAM_VUW_FilterPI: Invalid port number: " + sport
logger.error(errMsg)
return 1 # matched as if the filter does not exist
port = int(sport)
# Perform the matching.
client = ngamsPClient.ngamsPClient(host,
port,
timeOut=NGAMS_SOCK_TIMEOUT_DEF)
try:
if (fileVersion == -1):
fileVersion = 1
rest = client.get_status(NGAMS_STATUS_CMD,
pars=[["file_id", fileId],
["file_version", fileVersion]])
# since the queue will be sorted based on ingestion date, this will ensure the versions are sent by order:
# e.g. version1, version2, version3, otherwise, this method will have disordered versions sent
if (rest.getStatus().find(NGAMS_FAILURE) != -1):
return 1 # matched since file id does not exist
#if the same file id (with the latest version) is there already, check CRC
"""
xmlnode = rest.genXml(genDiskStatus = 1, genFileStatus = 1)
tgtCrc = xmlnode.getElementsByTagName('FileStatus')[0].attributes['Checksum'].value
cursorObj = srvObj.getDb().getFileInfoList('', fileId, fileVersion)
fileList = cursorObj.fetch(1)
srcCrc = None
for fileInfo in fileList:
srcCrc = fileInfo[10]
if (cursorObj):
del cursorObj
if (srcCrc and tgtCrc):
if (srcCrc != tgtCrc):
match = 1
else:
match = 1 # if no CRC information can be found, send the file regardless
"""
except Exception:
errMsg = "Error occurred during checking remote file status " +\
"ngamsGLEAM_VUW_FilterPI"
logger.exception(errMsg)
return 1 # matched as if the filter does not exist
#info(5, "filter return status = " + rest.getStatus())
#info(4, "filter match = " + str(match))
return match
