def performProcessing(srvObj, reqPropsObj, filename, mimeType):
"""
Carry out the processing requested.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
filename: File to be processed (string).
mimeType: Mime-type of file (string).
Returns: List with ngamsDppiStatus object
(list/ngamsDppiStatus objects).
"""
T = TRACE()
statusObjList = []
# Carry out the processing specified. If no processing is
# specified, we simply set the source file as the file to be send.
if (reqPropsObj.hasHttpPar("processing")):
dppi = reqPropsObj.getHttpPar("processing")
# Before starting to process, check if the specified DPPI
# is supported by this NG/AMS.
if dppi not in srvObj.getCfg().dppi_plugins:
errMsg = genLog("NGAMS_ER_ILL_DPPI", [dppi])
raise Exception, errMsg
# Invoke the DPPI.
logger.info("Invoking DPPI: %s to process file: %s", dppi, filename)
plugInMethod = loadPlugInEntryPoint(dppi)
statusObj = plugInMethod(srvObj, reqPropsObj, filename)
else:
logger.info("No processing requested - sending back file as is")
resultObj = ngamsDppiStatus.ngamsDppiResult(NGAMS_PROC_FILE, mimeType,
filename, filename)
statusObj = ngamsDppiStatus.ngamsDppiStatus().addResult(resultObj)
statusObjList.append(statusObj)
return statusObjList
def ngamsEsoArchDppi(srvObj,
reqPropsObj,
filename):
"""
This DPPI performs the processing neccessary for the files
requested from the ESO Archive (by the Data Requestor).
srvObj: Reference to instance of the NG/AMS Server
class (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
filename: Name of file to process (string).
Returns: DPPI return status object (ngamsDppiStatus).
"""
statusObj = ngamsDppiStatus.ngamsDppiStatus()
# Decompress the file if the last extension is "Z".
if (filename.split(".")[-1] == "Z"):
procFilename, procDir = ngamsPlugInApi.prepProcFile(srvObj.getCfg(),
filename)
exitCode, stdOut, stdErr = ngamsPlugInApi.\
execCmd("uncompress " + procFilename)
if (exitCode != 0):
errMsg = "ngamsEsoArchDppi: Problems during archiving! " +\
"Decompressing the file: " + filename + " failed. " +\
"Error message: " + str(stdErr)
raise Exception(errMsg)
resFilename = procFilename[0:-2]
else:
resFilename = filename
procDir = ""
mimeType = ngamsPlugInApi.determineMimeType(srvObj.getCfg(), resFilename)
resObj = ngamsDppiStatus.ngamsDppiResult(NGAMS_PROC_FILE, mimeType,
resFilename, resFilename, procDir)
statusObj.addResult(resObj)
return statusObj
def ngamsMWACortexStageDppi(srvObj,
reqPropsObj,
filename):
"""
This plugin stages the file from Tape to the disk, works ONLY in the iVEC Cortex environment.
srvObj: Reference to instance of the NG/AMS Server
class (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
filename: Name of file to process (string).
Returns: DPPI return status object (ngamsDppiStatus).
"""
#mimeType = ngamsPlugInApi.determineMimeType(srvObj.getCfg(), filename)
mimeType = "application/octet-stream" #hardcode this since our file extension is actually ".fits"
resultObj = ngamsDppiStatus.ngamsDppiResult(NGAMS_PROC_FILE, mimeType,
filename, filename)
statusObj = ngamsDppiStatus.ngamsDppiStatus().addResult(resultObj)
#procFilename, procDir = ngamsPlugInApi.prepProcFile(srvObj.getCfg(), filename)
cmd = "sls -D " + filename
t = ngamsPlugInApi.execCmd(cmd, -1)
exitCode = t[0]
if (exitCode != 0 or len(t) != 2):
logger.error("Fail to query the online/offline status for file %s", filename)
return statusObj #the sls -D command failed to execute, but retrieval might still go on, so just simply return empty result
offline = t[1].find('offline;')
if (offline != -1): # the file is offline, i.e. it is on tape
logger.debug("File %s is currently on tapes, staging it for retrieval...", filename)
cmd = "stage -w " + filename
t = ngamsPlugInApi.execCmd(cmd, -1) #stage it back to disk cache
logger.debug("File %s staging completed.", filename)
return statusObj
def ngasExtractFitsHdrDppi(srvObj, reqPropsObj, filename):
"""
This DPPI extracts the main header from a FITS file
requested from the ESO Archive.
srvObj: Reference to instance of the NG/AMS Server
class (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
filename: Name of file to process (string).
Returns: DPPI return status object (ngamsDppiStatus).
Side effect: This DPPI works directly on the archived file, since it is
read-only access.
SPECIFIC DOCUMENTATION:
This DPPI controls the call to the printhead module. If the
Example URL (single line):
http://ngasdev1:7777/RETRIEVE
?file_id=MIDI.2004-02-11T04:16:04.528&
processing=ngasExtractFitsHdrDppi&
processing_pars=header%3D99
The header parameter is optional, if not specified the primary header
is returned.
Valid values for the header parameter are numbers between 0 and 99. If
numbers are specified which are either outside the range or bigger than
the number of headers (including the primary) the primary header will be
returned. Headers are counted from 0 starting with the primary header. 99
is a special value as it returns all headers concatenated in a single file.
If 'xml=vo' is specified headers are returned using a slightly modified
VOTable (XML) format. If 'xml=xfits' is specified headers are returned
using the XFits (XML) format. struct=1 returns the structure of the FITS
file. tsv=1 returns the headers in a tab separated format suitable for
direct ingest into the header repository.
"""
logger.debug("Entering ngasExtractFitsHdrDppi() ...")
statusObj = ngamsDppiStatus.ngamsDppiStatus()
if (reqPropsObj.hasHttpPar("processing_pars")):
pars = ngamsPlugInApi.\
parseRawPlugInPars(reqPropsObj.getHttpPar("processing_pars"))
else:
# default is to extract the primary header
pars = {'header': 0}
logger.debug("ngasExtractFitsHdrDppi: %s %r", filename, pars)
PARS = set(['header', 'xml', 'skey', 'struct', 'tsv', 'check'])
# initial settings for printhead
xtract = 0
parse = 0
xmlVals = ['xfits', 'vo']
xmlfl = ''
skeyfl = 0
skey = 'END'
show = 0
struct = 0
tsv = 0
check = 0
mergefl = 0
hfl = 0
mode = 1
result = ''
err = ''
ext = 'hdr'
if pars.has_key('header'):
# extract a certain header: if value == 99 all headers are extracted,
# for any other value that header is extracted. headers are
# counted from 0
hfl = 1
struct = 0
show = pars['header']
try:
head = int(show)
except:
err = "ngasExtractFitsHdrDppi: Invalid type for header " +\
"parameter. Should be int"
if head < 0 or head > 99:
err = "ngasExtractFitsHdrDppi: Invalid value specified for " +\
"header parameter."
if pars.has_key('xml'):
# if this key exists we do a conversion to XFits XML.
struct = 0
if pars['xml'] in xmlVals:
xmlfl = pars['xml']
else:
err = "ngasExtractFitsHdrDppi: Invalid value for xml " +\
"parameter. Should be 'vo|xfits': "+ pars['xml']
ext = 'xml'
if pars.has_key('skey'):
# extract just one keyword. CAUTION: No checking done!
skey = pars['skey'].strip()
skeyfl = 1
keyParts = skey.split()
ext = 'txt'
head = int(head)
if head < 0: head = 0
if ((not re.match('[a-zA-Z]', skey[0]))
or (len(keyParts) > 1 and keyParts[0] != 'HIERARCH')
or (len(keyParts[0]) > 8)):
err = "ngasExtractFitsHdrDppi: Invalid value for skey " +\
"parameter specified. Must be a valid FITS keyword:",\
skey
if pars.has_key('struct'):
# return only the structure of the FITS file. Value of the
# parameter is ignored
head = -99
struct = 1
ext = 'txt'
if pars.has_key('tsv'):
# extract header in tsv format. Parameter value is ignored
struct = 1
tsv = 1
ext = 'txt'
head = int(head)
if head < 0: head = 0
if pars.has_key('check'):
# head structure and calculate the checksum of the data part.
head = -99
struct = 1
check = 1
# printhead supports a list of files, but here we only use one
fils = [filename]
base = os.path.basename(filename)
pos = base.rfind('.fits')
file_id = base[:pos]
if len(set(pars) - PARS) != 0: # detect unsupported parameters
xpars = set(pars) - PARS
err = "ngasExtractFitsHdrDppi: Unsupported option(s): %s\n" + \
"Valid options are:\n" + \
"header=<number> where number is an integer between 0 " +\
"and max(extension)-1 or 99 (for all)\n" +\
"xml=<format>, where format is [vo|xfits]\n" +\
"struct=1\n" +\
"skey=<FITS keyword>, should be a valid keyword, crude " +\
"checking is done\n" + \
"tsv=1\n\n" + \
"combinations are allowed and should be separated by " +\
"a comma character.\n"
err = err % xpars
ext = 'txt'
file_id = 'error'
result = '' # initialize result string
if err != '':
result = err
if result == '':
for f in fils:
cmd = constructCommand(f, head, struct, skey, tsv, xmlfl, mode,
check)
logger.debug('Executing command: %s', cmd)
stat, result, _ = execCmd(cmd)
if stat != 0:
errMsg = "Processing of header for file %s failed: %s" % (
filename, result)
raise Exception(errMsg)
resFilename = file_id + "." + ext
try:
# mime-type guessing does not work sometimes, we force it in that case.
mimeType = ngamsPlugInApi.determineMimeType(srvObj.getCfg(),
resFilename)
except:
if ext == 'xml':
mimeType = 'text/xml'
else:
mimeType = 'text/ascii'
resObj = ngamsDppiStatus.ngamsDppiResult(NGAMS_PROC_DATA, mimeType, result,
resFilename, '')
statusObj.addResult(resObj)
logger.debug("Leaving ngasExtractFitsHdrDppi() ...")
return statusObj
def ngamsExtractAlmaMultipart(srvObj, reqPropsObj, file_id):
"""
This DPPI extracts one part of a multipart/related message
requested from the ALMA Archive.
srvObj: Reference to instance of the NG/AMS Server
class (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
file_id: Name of file to process (string).
Returns: DPPI return status object (ngamsDppiStatus).
Side effect: This DPPI works directly on the archived file, since it is
read-only access.
SPECIFIC DOCUMENTATION:
This DPPI extracts one part of an ALMA multipart related message and
returns a complete, self-consistent message containing a XML header
and the requested cid. This version deals with VOTable headers and
returns a new VOTable header containing only the RESOURCE element
describing the requested (by cid) part. If no XML header can be found
or if the XML header is not a VOTable header this plugin returns just the
requested part, without any header.
Example URL (single line):
http://ngasdev1:7777/RETRIEVE
?file_id=X01/X7/X42&
processing=ngamsAlmaMultipart&
processing_pars='cid=<cid>'
"""
statusObj = ngamsDppiStatus.ngamsDppiStatus()
cpart = 1
pars = 0 # initialize pars
if (reqPropsObj.hasHttpPar("processing_pars")):
pars = ngamsPlugInApi.parseRawPlugInPars(\
reqPropsObj.getHttpPar("processing_pars"))
if pars and not pars.has_key('cid'):
ext = '.cid'
cpart = 1 # first part only by default
elif pars and pars.has_key(
'cid'): # if processing_par 'cid' exists check its contents
pass
else:
pass
resFilename = file_id + ext
try:
mimeType = ngamsPlugInApi.determineMimeType(srvObj.getCfg(),\
resFilename)
except:
pass
if ext == '.xml':
mimeType = 'text/xml'
else:
mimeType = 'multipart/related'
resObj = ngamsDppiStatus.ngamsDppiResult(NGAMS_PROC_DATA, mimeType, head,
resFilename, '')
statusObj.addResult(resObj)
return statusObj
def ngamsTestDppi1(srvObj, reqPropsObj, filename):
"""
This test DPPI extracts the main header from a FITS file requested from the
ESO Archive.
Depending on the the DPPI Plug-In Parameters it stores the result in a file
or in a buffer in memory.
This is made to work on small FITS files.
srvObj: Reference to instance of the NG/AMS Server
class (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
filename: Name of file to process (string).
Returns: DPPI return status object (ngamsDppiStatus).
"""
statusObj = ngamsDppiStatus.ngamsDppiStatus()
# Uncompress the file.
procFile, procDir = ngamsPlugInApi.prepProcFile(srvObj.getCfg(), filename)
subprocess.check_call(['gunzip', procFile])
procFile = procFile[0:procFile.rfind(".")]
# dpallot: fold fails miserably on Mac when dealing with binary files
#
# Process the output file.
#stat, out = commands.getstatusoutput("fold %s" % procFile)
#if stat:
# raise Exception('Problem while folding %s: %s' % (procFile, out))
head = []
with open(procFile, 'rb') as f:
while True:
line = f.read(80)
assert (line and len(line) == 80)
head.append(line)
head.append(b'\n')
if b'END' in line:
break
mimeType = "TEST-MIME-TYPE"
rawPiPars = srvObj.getCfg().dppi_plugins["ngamsTest.ngamsTestDppi1"].pars
cfgParDic = ngamsPlugInApi.parseRawPlugInPars(rawPiPars)
head.append(b"\n\nConfiguration Parameters:\n")
parList = list(cfgParDic)
parList.sort()
for par in parList:
head.append(six.b("%s=%s\n" % (par, cfgParDic[par])))
head.append(b"\nParameters Transferred:\n")
httpParsDic = reqPropsObj.getHttpParsDic()
httpPars = list(httpParsDic)
httpPars.sort()
for httpPar in httpPars:
head.append(six.b("%s=%s\n" % (httpPar, httpParsDic[httpPar])))
head.append(b"\nEOF\n")
buf = b''.join(head)
# Generate status.
if (cfgParDic["TARGET"] == "FILE"):
outFile = procFile + "_ngamsTestDppi1"
with open(outFile, "ab") as fo:
fo.write(buf)
resObj = ngamsDppiStatus.ngamsDppiResult(NGAMS_PROC_FILE, mimeType,
outFile, filename, procDir)
else:
resObj = ngamsDppiStatus.ngamsDppiResult(NGAMS_PROC_DATA, mimeType,
buf, filename)
statusObj.addResult(resObj)
return statusObj