コード例 #1
0
def performProcessing(srvObj, reqPropsObj, filename, mimeType):
    """
    Carry out the processing requested.

    srvObj:       Reference to NG/AMS server class object (ngamsServer).

    reqPropsObj:  Request Property object to keep track of actions done
                  during the request handling (ngamsReqProps).

    filename:     File to be processed (string).

    mimeType:     Mime-type of file (string).

    Returns:      List with ngamsDppiStatus object
                  (list/ngamsDppiStatus objects).
    """
    T = TRACE()

    statusObjList = []

    # Carry out the processing specified. If no processing is
    # specified, we simply set the source file as the file to be send.
    if (reqPropsObj.hasHttpPar("processing")):
        dppi = reqPropsObj.getHttpPar("processing")
        # Before starting to process, check if the specified DPPI
        # is supported by this NG/AMS.
        if dppi not in srvObj.getCfg().dppi_plugins:
            errMsg = genLog("NGAMS_ER_ILL_DPPI", [dppi])
            raise Exception, errMsg
        # Invoke the DPPI.
        logger.info("Invoking DPPI: %s to process file: %s", dppi, filename)
        plugInMethod = loadPlugInEntryPoint(dppi)
        statusObj = plugInMethod(srvObj, reqPropsObj, filename)
    else:
        logger.info("No processing requested - sending back file as is")
        resultObj = ngamsDppiStatus.ngamsDppiResult(NGAMS_PROC_FILE, mimeType,
                                                    filename, filename)
        statusObj = ngamsDppiStatus.ngamsDppiStatus().addResult(resultObj)
    statusObjList.append(statusObj)

    return statusObjList
コード例 #2
0
def ngamsEsoArchDppi(srvObj,
                     reqPropsObj,
                     filename):
    """
    This DPPI performs the processing neccessary for the files
    requested from the ESO Archive (by the Data Requestor).

    srvObj:        Reference to instance of the NG/AMS Server
                   class (ngamsServer).

    reqPropsObj:   NG/AMS request properties object (ngamsReqProps).

    filename:      Name of file to process (string).

    Returns:       DPPI return status object (ngamsDppiStatus).
    """
    statusObj = ngamsDppiStatus.ngamsDppiStatus()

    # Decompress the file if the last extension is "Z".
    if (filename.split(".")[-1] == "Z"):
        procFilename, procDir = ngamsPlugInApi.prepProcFile(srvObj.getCfg(),
                                                            filename)
        exitCode, stdOut, stdErr = ngamsPlugInApi.\
                                   execCmd("uncompress " + procFilename)
        if (exitCode != 0):
            errMsg = "ngamsEsoArchDppi: Problems during archiving! " +\
                     "Decompressing the file: " + filename + " failed. " +\
                     "Error message: " + str(stdErr)
            raise Exception(errMsg)
        resFilename = procFilename[0:-2]
    else:
        resFilename = filename
        procDir = ""
    mimeType = ngamsPlugInApi.determineMimeType(srvObj.getCfg(), resFilename)
    resObj = ngamsDppiStatus.ngamsDppiResult(NGAMS_PROC_FILE, mimeType,
                                             resFilename, resFilename, procDir)
    statusObj.addResult(resObj)

    return statusObj
コード例 #3
0
def ngamsMWACortexStageDppi(srvObj,
                           reqPropsObj,
                           filename):
    """
    This plugin stages the file from Tape to the disk, works ONLY in the iVEC Cortex environment.

    srvObj:        Reference to instance of the NG/AMS Server
                   class (ngamsServer).

    reqPropsObj:   NG/AMS request properties object (ngamsReqProps).

    filename:      Name of file to process (string).

    Returns:       DPPI return status object (ngamsDppiStatus).
    """
    #mimeType = ngamsPlugInApi.determineMimeType(srvObj.getCfg(), filename)
    mimeType = "application/octet-stream" #hardcode this since our file extension is actually ".fits"
    resultObj = ngamsDppiStatus.ngamsDppiResult(NGAMS_PROC_FILE, mimeType,
                                                    filename, filename)
    statusObj = ngamsDppiStatus.ngamsDppiStatus().addResult(resultObj)

    #procFilename, procDir = ngamsPlugInApi.prepProcFile(srvObj.getCfg(), filename)
    cmd = "sls -D " + filename
    t = ngamsPlugInApi.execCmd(cmd, -1)
    exitCode = t[0]
    if (exitCode != 0 or len(t) != 2):
        logger.error("Fail to query the online/offline status for file %s", filename)
        return statusObj #the sls -D command failed to execute, but retrieval might still go on, so just simply return empty result

    offline = t[1].find('offline;')

    if (offline != -1): # the file is offline, i.e. it is on tape
        logger.debug("File %s is currently on tapes, staging it for retrieval...", filename)
        cmd = "stage -w " + filename
        t = ngamsPlugInApi.execCmd(cmd, -1) #stage it back to disk cache
        logger.debug("File %s staging completed.", filename)

    return statusObj
コード例 #4
0
def ngasExtractFitsHdrDppi(srvObj, reqPropsObj, filename):
    """
    This DPPI extracts the main header from a FITS file
    requested from the ESO Archive.

    srvObj:        Reference to instance of the NG/AMS Server
                   class (ngamsServer).

    reqPropsObj:   NG/AMS request properties object (ngamsReqProps).

    filename:      Name of file to process (string).

    Returns:       DPPI return status object (ngamsDppiStatus).

    Side effect:   This DPPI works directly on the archived file, since it is
                   read-only access.

    SPECIFIC DOCUMENTATION:
    This DPPI controls the call to the printhead module. If the

    Example URL (single line):

	http://ngasdev1:7777/RETRIEVE
		?file_id=MIDI.2004-02-11T04:16:04.528&
		processing=ngasExtractFitsHdrDppi&
		processing_pars=header%3D99

    The header parameter is optional, if not specified the primary header
    is returned.

    Valid values for the header parameter are numbers between 0 and 99. If
    numbers are specified which are either outside the range or bigger than
    the number of headers (including the primary) the primary header will be
    returned. Headers are counted from 0 starting with the primary header. 99
    is a special value as it returns all headers concatenated in a single file.

    If 'xml=vo' is specified headers are returned using a slightly modified
    VOTable (XML) format. If 'xml=xfits' is specified headers are returned
    using the XFits (XML) format. struct=1 returns the structure of the FITS
    file. tsv=1 returns the headers in a tab separated format suitable for
    direct ingest into the header repository.
    """
    logger.debug("Entering ngasExtractFitsHdrDppi() ...")

    statusObj = ngamsDppiStatus.ngamsDppiStatus()

    if (reqPropsObj.hasHttpPar("processing_pars")):
        pars = ngamsPlugInApi.\
               parseRawPlugInPars(reqPropsObj.getHttpPar("processing_pars"))
    else:
        # default is to extract the primary header
        pars = {'header': 0}

    logger.debug("ngasExtractFitsHdrDppi: %s %r", filename, pars)

    PARS = set(['header', 'xml', 'skey', 'struct', 'tsv', 'check'])

    # initial settings for printhead
    xtract = 0
    parse = 0
    xmlVals = ['xfits', 'vo']
    xmlfl = ''
    skeyfl = 0
    skey = 'END'
    show = 0
    struct = 0
    tsv = 0
    check = 0
    mergefl = 0
    hfl = 0
    mode = 1

    result = ''
    err = ''
    ext = 'hdr'

    if pars.has_key('header'):
        # extract a certain header: if value == 99 all headers are extracted,
        # for any other value that header is extracted. headers are
        # counted from 0
        hfl = 1
        struct = 0
        show = pars['header']
        try:
            head = int(show)
        except:
            err = "ngasExtractFitsHdrDppi: Invalid type for header " +\
                  "parameter. Should be int"
        if head < 0 or head > 99:
            err = "ngasExtractFitsHdrDppi: Invalid value specified for " +\
                  "header parameter."

    if pars.has_key('xml'):
        # if this key exists we do a conversion to XFits XML.
        struct = 0
        if pars['xml'] in xmlVals:
            xmlfl = pars['xml']
        else:
            err = "ngasExtractFitsHdrDppi: Invalid value for xml " +\
                  "parameter. Should be 'vo|xfits': "+ pars['xml']
        ext = 'xml'

    if pars.has_key('skey'):
        # extract just one keyword. CAUTION: No checking done!
        skey = pars['skey'].strip()
        skeyfl = 1
        keyParts = skey.split()
        ext = 'txt'
        head = int(head)
        if head < 0: head = 0
        if ((not re.match('[a-zA-Z]', skey[0]))
                or (len(keyParts) > 1 and keyParts[0] != 'HIERARCH')
                or (len(keyParts[0]) > 8)):
            err = "ngasExtractFitsHdrDppi: Invalid value for skey " +\
                  "parameter specified. Must be a valid FITS keyword:",\
                  skey

    if pars.has_key('struct'):
        # return only the structure of the FITS file. Value of the
        # parameter is ignored
        head = -99
        struct = 1
        ext = 'txt'

    if pars.has_key('tsv'):
        # extract header in tsv format. Parameter value is ignored
        struct = 1
        tsv = 1
        ext = 'txt'
        head = int(head)
        if head < 0: head = 0

    if pars.has_key('check'):
        # head structure and calculate the checksum of the data part.
        head = -99
        struct = 1
        check = 1

    # printhead supports a list of files, but here we only use one
    fils = [filename]
    base = os.path.basename(filename)
    pos = base.rfind('.fits')
    file_id = base[:pos]

    if len(set(pars) - PARS) != 0:  # detect unsupported parameters
        xpars = set(pars) - PARS
        err   = "ngasExtractFitsHdrDppi: Unsupported option(s): %s\n" + \
                "Valid options are:\n" + \
                "header=<number> where number is an integer between 0 " +\
                "and max(extension)-1 or 99 (for all)\n" +\
                "xml=<format>, where format is [vo|xfits]\n" +\
                "struct=1\n" +\
                "skey=<FITS keyword>, should be a valid keyword, crude " +\
                "checking is done\n" + \
                "tsv=1\n\n" + \
                "combinations are allowed and should be separated by " +\
                "a comma character.\n"
        err = err % xpars
        ext = 'txt'
        file_id = 'error'

    result = ''  # initialize result string

    if err != '':
        result = err

    if result == '':
        for f in fils:
            cmd = constructCommand(f, head, struct, skey, tsv, xmlfl, mode,
                                   check)
            logger.debug('Executing command: %s', cmd)
            stat, result, _ = execCmd(cmd)
            if stat != 0:
                errMsg = "Processing of header for file %s failed: %s" % (
                    filename, result)
                raise Exception(errMsg)

    resFilename = file_id + "." + ext
    try:
        # mime-type guessing does not work sometimes, we force it in that case.
        mimeType = ngamsPlugInApi.determineMimeType(srvObj.getCfg(),
                                                    resFilename)
    except:
        if ext == 'xml':
            mimeType = 'text/xml'
        else:
            mimeType = 'text/ascii'

    resObj = ngamsDppiStatus.ngamsDppiResult(NGAMS_PROC_DATA, mimeType, result,
                                             resFilename, '')
    statusObj.addResult(resObj)

    logger.debug("Leaving ngasExtractFitsHdrDppi() ...")

    return statusObj
コード例 #5
0
def ngamsExtractAlmaMultipart(srvObj, reqPropsObj, file_id):
    """
    This DPPI extracts one part of a multipart/related message
    requested from the ALMA Archive.

    srvObj:        Reference to instance of the NG/AMS Server
                   class (ngamsServer).

    reqPropsObj:   NG/AMS request properties object (ngamsReqProps).

    file_id:      Name of file to process (string).

    Returns:       DPPI return status object (ngamsDppiStatus).

    Side effect:   This DPPI works directly on the archived file, since it is
                   read-only access.

    SPECIFIC DOCUMENTATION:

        This DPPI extracts one part of an ALMA multipart related message and
        returns a complete, self-consistent message containing a XML header
        and the requested cid. This version deals with VOTable headers and
        returns a new VOTable header containing only the RESOURCE element
        describing the requested (by cid) part. If no XML header can be found
        or if the XML header is not a VOTable header this plugin returns just the
        requested part, without any header.

    Example URL (single line):

    http://ngasdev1:7777/RETRIEVE
        ?file_id=X01/X7/X42&
        processing=ngamsAlmaMultipart&
        processing_pars='cid=<cid>'


    """
    statusObj = ngamsDppiStatus.ngamsDppiStatus()

    cpart = 1
    pars = 0  # initialize pars
    if (reqPropsObj.hasHttpPar("processing_pars")):
        pars = ngamsPlugInApi.parseRawPlugInPars(\
        reqPropsObj.getHttpPar("processing_pars"))
    if pars and not pars.has_key('cid'):
        ext = '.cid'
        cpart = 1  # first part only by default
    elif pars and pars.has_key(
            'cid'):  # if processing_par 'cid' exists check its contents

        pass
    else:
        pass

    resFilename = file_id + ext
    try:
        mimeType = ngamsPlugInApi.determineMimeType(srvObj.getCfg(),\
         resFilename)
    except:
        pass
    if ext == '.xml':
        mimeType = 'text/xml'
    else:
        mimeType = 'multipart/related'

    resObj = ngamsDppiStatus.ngamsDppiResult(NGAMS_PROC_DATA, mimeType, head,
                                             resFilename, '')
    statusObj.addResult(resObj)

    return statusObj
コード例 #6
0
ファイル: ngamsTestDppi1.py プロジェクト: 33bunny/ngas
def ngamsTestDppi1(srvObj, reqPropsObj, filename):
    """
    This test DPPI extracts the main header from a FITS file requested from the
    ESO Archive.

    Depending on the the DPPI Plug-In Parameters it stores the result in a file
    or in a buffer in memory.

    This is made to work on small FITS files.

    srvObj:        Reference to instance of the NG/AMS Server
                   class (ngamsServer).

    reqPropsObj:   NG/AMS request properties object (ngamsReqProps).

    filename:      Name of file to process (string).

    Returns:       DPPI return status object (ngamsDppiStatus).
    """
    statusObj = ngamsDppiStatus.ngamsDppiStatus()

    # Uncompress the file.
    procFile, procDir = ngamsPlugInApi.prepProcFile(srvObj.getCfg(), filename)
    subprocess.check_call(['gunzip', procFile])
    procFile = procFile[0:procFile.rfind(".")]

    # dpallot: fold fails miserably on Mac when dealing with binary files
    #
    # Process the output file.
    #stat, out = commands.getstatusoutput("fold %s" % procFile)
    #if stat:
    #    raise Exception('Problem while folding %s: %s' % (procFile, out))

    head = []
    with open(procFile, 'rb') as f:
        while True:
            line = f.read(80)
            assert (line and len(line) == 80)
            head.append(line)
            head.append(b'\n')
            if b'END' in line:
                break

    mimeType = "TEST-MIME-TYPE"
    rawPiPars = srvObj.getCfg().dppi_plugins["ngamsTest.ngamsTestDppi1"].pars
    cfgParDic = ngamsPlugInApi.parseRawPlugInPars(rawPiPars)

    head.append(b"\n\nConfiguration Parameters:\n")
    parList = list(cfgParDic)
    parList.sort()
    for par in parList:
        head.append(six.b("%s=%s\n" % (par, cfgParDic[par])))

    head.append(b"\nParameters Transferred:\n")
    httpParsDic = reqPropsObj.getHttpParsDic()
    httpPars = list(httpParsDic)
    httpPars.sort()
    for httpPar in httpPars:
        head.append(six.b("%s=%s\n" % (httpPar, httpParsDic[httpPar])))
    head.append(b"\nEOF\n")

    buf = b''.join(head)
    # Generate status.
    if (cfgParDic["TARGET"] == "FILE"):
        outFile = procFile + "_ngamsTestDppi1"
        with open(outFile, "ab") as fo:
            fo.write(buf)
        resObj = ngamsDppiStatus.ngamsDppiResult(NGAMS_PROC_FILE, mimeType,
                                                 outFile, filename, procDir)
    else:
        resObj = ngamsDppiStatus.ngamsDppiResult(NGAMS_PROC_DATA, mimeType,
                                                 buf, filename)
    statusObj.addResult(resObj)
    return statusObj