def ngams_generic(ngams_server, request_properties):
"""
Data Archiving Plug-In to handle archiving of SDM multipart related message files containing ALMA UIDs in the
Content-Location mime parameter or any other kind of file
:param ngams_server: Reference to NG/AMS Server Object (ngamsServer)
:param request_properties: NG/AMS request properties object (ngamsReqProps)
:return: Standard NG/AMS Data Archiving Plug-In Status as generated by: ngamsPlugInApi.genDapiSuccessStat()
(ngamsDapiStatus)
"""
logger.info("Mirroring plug-in handling data for file: %s", os.path.basename(request_properties.getFileUri()))
# Create staging file
disk_info = request_properties.getTargDiskInfo()
staging_filename = request_properties.getStagingFilename()
# request_properties format: /MIRRARCHIVE?mime_type=application/x-tar&filename=...
file_format = request_properties.getMimeType()
if not file_format:
raise Exception("mime_type parameter not specified in MIRRARCHIVE request")
# Example of file URI format:
# http://ngas01.org:7777/RETRIEVE?disk_id=59622720f79296473f6106c15e5c2240&host_id=ngas01:7777&quick_location=1&file_version=1&file_id=backup.2011-02-02T22:01:59.tar
file_id = request_properties.fileinfo["fileId"]
file_version = request_properties.fileinfo["fileVersion"]
# Specific treatment depending on the mime-type
if file_format.find("multipart") >= 0 or file_format.find("multialma") >= 0:
logger.debug("Mirroring plug-in applying specific treatment for multipart/multialma mime file")
file_id, final_filename, file_format = specific_treatment(staging_filename)
else:
final_filename = file_id
logger.debug("Mirroring plug-in processing request for file with URI %s, file_format=%s, file_id=%s, "
"file_version=%s, final_filename=%s", request_properties.getFileUri(), file_format, file_id,
file_version, final_filename)
try:
# Compression parameters
uncompressed_size = ngamsPlugInApi.getFileSize(staging_filename)
compression = ""
today = ngamsCore.toiso8601(fmt=ngamsCore.FMT_DATE_ONLY)
relative_path, relative_filename, complete_filename, file_exists = \
generate_file_info(ngams_server.getCfg(), disk_info, staging_filename, file_version, final_filename, [today])
# Make sure the format is defined
if not file_format:
file_format = ngamsPlugInApi.determineMimeType(ngams_server.getCfg(), staging_filename)
file_size = ngamsPlugInApi.getFileSize(staging_filename)
return ngamsPlugInApi.genDapiSuccessStat(disk_info.getDiskId(), relative_filename, file_id, file_version,
file_format, file_size, uncompressed_size, compression, relative_path,
disk_info.getSlotId(), file_exists, complete_filename)
except Exception as e:
raise Exception(genLog("NGAMS_ER_DAPI_BAD_FILE",
[staging_filename, PLUGIN_ID, "Problem processing file in staging area: " + str(e)]))
def ngamsSdmMultipart(ngams_server, request_properties):
"""
Data Archiving Plug-In to handle archiving of SDM multipart related message files containing ALMA UIDs in the
Content-Location mime parameter
:param ngams_server: Reference to NG/AMS Server Object (ngamsServer)
:param request_properties: NG/AMS request properties object (ngamsReqProps)
:return: Standard NG/AMS Data Archiving Plug-In Status as generated by: ngamsPlugInApi.genDapiSuccessStat()
(ngamsDapiStatus)
"""
# For now the exception handling is pretty basic:
# If something goes wrong during the handling it is tried to move the temporary file to the bad-files directory
logger.info("SDM multipart plug-in handling data for file: %s", os.path.basename(request_properties.getFileUri()))
disk_info = request_properties.getTargDiskInfo()
staging_filename = request_properties.getStagingFilename()
file_id, final_filename, file_format = specific_treatment(staging_filename)
if request_properties.hasHttpPar("file_id"):
file_id = request_properties.getHttpPar("file_id")
#if request_properties.hasHttpPar("file_version"):
# file_version = request_properties.getHttpPar("file_version")
logger.debug("SDM multipart plug-in processing request for file with URI %s, file_format=%s, file_id=%s, "
"final_filename=%s", request_properties.getFileUri(), file_format, file_id, final_filename)
try:
# Compression parameters
uncompressed_size = ngamsPlugInApi.getFileSize(staging_filename)
compression = ""
# Remember to update the temporary file name in the request properties object
request_properties.setStagingFilename(staging_filename)
today = ngamsCore.toiso8601(fmt=ngamsCore.FMT_DATE_ONLY)
file_version, relative_path, relative_filename, complete_filename, file_exists = \
ngamsPlugInApi.genFileInfo(ngams_server.getDb(), ngams_server.getCfg(), request_properties, disk_info,
staging_filename, file_id, final_filename, [today])
# Make sure the format is defined
if not file_format:
file_format = ngamsPlugInApi.determineMimeType(ngams_server.getCfg(), staging_filename)
file_size = ngamsPlugInApi.getFileSize(staging_filename)
return ngamsPlugInApi.genDapiSuccessStat(disk_info.getDiskId(), relative_filename, file_id, file_version,
file_format, file_size, uncompressed_size, compression, relative_path,
disk_info.getSlotId(), file_exists, complete_filename)
except Exception as e:
raise Exception(genLog("NGAMS_ER_DAPI_BAD_FILE",
[staging_filename, PLUGIN_ID, "Problem processing file in staging area: " + str(e)]))
def ngamsMimeTypeFilterPI(srvObj,
plugInPars,
filename,
fileId,
fileVersion=-1,
reqPropsObj=None):
"""
Example Filter Plug-In used to filter on a given mime-type. In case the
file referenced has the mime-type as specified in the plug-in parameters,
the file being tested is selected.
srvObj: Reference to NG/AMS Server Object (ngamsServer).
plugInPars: Parameters to take into account for the plug-in
execution (string).
fileId: File ID for file to test (string).
filename: Filename of (complete) (string).
fileVersion: Version of file to test (integer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: 0 if the file does not match, 1 if it matches the
conditions (integer/0|1).
"""
match = 0
# Parse plug-in parameters.
parDic = []
pars = ""
if ((plugInPars != "") and (plugInPars != None)):
pars = plugInPars
elif (reqPropsObj != None):
if (reqPropsObj.hasHttpPar("plug_in_pars")):
pars = reqPropsObj.getHttpPar("plug_in_pars")
parDic = ngamsPlugInApi.parseRawPlugInPars(pars)
if (not parDic.has_key("mime_types")):
errMsg = "ngamsMimeTypeFilterPI: Missing Plug-In Parameter: " +\
"mime_types"
raise Exception, errMsg
# Perform the matching.
refMimeTypes = parDic["mime_types"].split("|")
actMimeType = ngamsPlugInApi.determineMimeType(srvObj.getCfg(), filename)
for mt in refMimeTypes:
if (actMimeType == mt.strip()): match = 1
return match
def ngamsEsoArchDppi(srvObj,
reqPropsObj,
filename):
"""
This DPPI performs the processing neccessary for the files
requested from the ESO Archive (by the Data Requestor).
srvObj: Reference to instance of the NG/AMS Server
class (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
filename: Name of file to process (string).
Returns: DPPI return status object (ngamsDppiStatus).
"""
statusObj = ngamsDppiStatus.ngamsDppiStatus()
# Decompress the file if the last extension is "Z".
if (filename.split(".")[-1] == "Z"):
procFilename, procDir = ngamsPlugInApi.prepProcFile(srvObj.getCfg(),
filename)
exitCode, stdOut, stdErr = ngamsPlugInApi.\
execCmd("uncompress " + procFilename)
if (exitCode != 0):
errMsg = "ngamsEsoArchDppi: Problems during archiving! " +\
"Decompressing the file: " + filename + " failed. " +\
"Error message: " + str(stdErr)
raise Exception(errMsg)
resFilename = procFilename[0:-2]
else:
resFilename = filename
procDir = ""
mimeType = ngamsPlugInApi.determineMimeType(srvObj.getCfg(), resFilename)
resObj = ngamsDppiStatus.ngamsDppiResult(NGAMS_PROC_FILE, mimeType,
resFilename, resFilename, procDir)
statusObj.addResult(resObj)
return statusObj
def ngamsAlmaMultipart(srvObj,
reqPropsObj):
"""
Data Archiving Plug-In to handle archiving of ALMA multipart related
message files containing ALMA UIDs.
srvObj: Reference to NG/AMS Server Object (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: Standard NG/AMS Data Archiving Plug-In Status
as generated by: ngamsPlugInApi.genDapiSuccessStat()
(ngamsDapiStatus).
"""
# For now the exception handling is pretty basic:
# If something goes wrong during the handling it is tried to
# move the temporary file to the Bad Files Area of the disk.
logger.info("Plug-In handling data for file: " +
os.path.basename(reqPropsObj.getFileUri()))
diskInfo = reqPropsObj.getTargDiskInfo()
stagingFilename = reqPropsObj.getStagingFilename()
ext = os.path.splitext(stagingFilename)[1][1:]
fo = open(stagingFilename, "r")
(fileId, finalName, format) = specificTreatment(fo)
fo.close()
try:
# Compress the file.
uncomprSize = ngamsPlugInApi.getFileSize(stagingFilename)
compression = ""
# info(2,"Compressing file using: %s ..." % compression)
# exitCode, stdOut = ngamsPlugInApi.execCmd("%s %s" %\
# (compression,
# stagingFilename))
# if (exitCode != 0):
# errMsg = _PLUGIN_ID+": Problems during archiving! " +\
# "Compressing the file failed"
# raise Exception, errMsg
# stagingFilename = stagingFilename + ".Z"
# Remember to update the Temporary Filename in the Request
# Properties Object.
reqPropsObj.setStagingFilename(stagingFilename)
# info(2,"File compressed")
# ToDo: Handling of non-existing fileId
# if (fileId == -1):
# fileId = ngamsPlugInApi.genNgasId(srvObj.getCfg())
date = toiso8601(local=True, fmt=FMT_DATE_ONLY)
fileVersion, relPath, relFilename,\
complFilename, fileExists =\
ngamsPlugInApi.genFileInfo(srvObj.getDb(),
srvObj.getCfg(),
reqPropsObj, diskInfo,
stagingFilename, fileId,
finalName, [date])
# Generate status.
logger.debug("Generating status ...")
if not format:
format = ngamsPlugInApi.determineMimeType(srvObj.getCfg(),
stagingFilename)
fileSize = ngamsPlugInApi.getFileSize(stagingFilename)
return ngamsPlugInApi.genDapiSuccessStat(diskInfo.getDiskId(),
relFilename,
fileId, fileVersion, format,
fileSize, uncomprSize,
compression, relPath,
diskInfo.getSlotId(),
fileExists, complFilename)
except Exception as e:
errMsg = genLog("NGAMS_ER_DAPI_BAD_FILE", [os.path.basename(stagingFilename),
_PLUGIN_ID, str(e)])
raise Exception(errMsg)
def compressFile(srvObj, reqPropsObj, parDic):
"""
Compress the file if required.
srvObj: Reference to NG/AMS Server Object (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
parDic: Dictionary with parameters for the DAPI. This is generated
with ngamsPlugInApi.parseDapiPlugInPars() (Dictionary).
Returns: Tupe containing uncompressed filesize, archived filesize
and the format (mime-type) of the resulting data file and
the compression method (NONE if the file is not compressed),
finally, the extension added by the compression if any
(tuple).
"""
stFn = reqPropsObj.getStagingFilename()
# If a compression application is specified, apply this.
uncomprSize = ngamsPlugInApi.getFileSize(stFn)
comprExt = ""
if (parDic[COMPRESSION]):
logger.debug("Compressing file using: %s ...", parDic[COMPRESSION])
compCmd = "%s %s" % (parDic[COMPRESSION], stFn)
compress_start = time.time()
logger.debug("Compressing file with command: %s", compCmd)
exitCode, stdOut = ngamsPlugInApi.execCmd(compCmd)
#if (exitCode != 0):
# msg ="Problems during archiving! Compressing the file failed. " +\
# "Error: %s" % str(stdOut).replace("/n", " ")
# raise Exception, msg
# If the compression fails, assume that it is because the file is not
# compressible (although it could also be due to lack of disk space).
if (exitCode == 0):
if (parDic[COMPRESSION_EXT]):
stFn = stFn + "." + parDic[COMPRESSION_EXT]
comprExt = parDic[COMPRESSION_EXT]
# Remember to update Staging Filename in the Request Properties
# Object.
reqPropsObj.setStagingFilename(stFn)
# Handle mime-type
if (parDic[TARG_MIME_TYPE]):
format = parDic[TARG_MIME_TYPE]
else:
format = ngamsPlugInApi.determineMimeType(
srvObj.getCfg(), stFn)
compression = parDic[COMPRESSION]
logger.debug("File compressed. Time: %.3fs",
time.time() - compress_start)
else:
# Carry on with the original file. We take the original mime-type
# as the target mime-type.
format = reqPropsObj.getMimeType()
compression = NO_COMPRESSION
else:
# Handle mime-type
if (parDic[TARG_MIME_TYPE]):
format = parDic[TARG_MIME_TYPE]
else:
format = reqPropsObj.getMimeType()
compression = NO_COMPRESSION
archFileSize = ngamsPlugInApi.getFileSize(reqPropsObj.getStagingFilename())
return uncomprSize, archFileSize, format, compression, comprExt
def ngasExtractFitsHdrDppi(srvObj, reqPropsObj, filename):
"""
This DPPI extracts the main header from a FITS file
requested from the ESO Archive.
srvObj: Reference to instance of the NG/AMS Server
class (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
filename: Name of file to process (string).
Returns: DPPI return status object (ngamsDppiStatus).
Side effect: This DPPI works directly on the archived file, since it is
read-only access.
SPECIFIC DOCUMENTATION:
This DPPI controls the call to the printhead module. If the
Example URL (single line):
http://ngasdev1:7777/RETRIEVE
?file_id=MIDI.2004-02-11T04:16:04.528&
processing=ngasExtractFitsHdrDppi&
processing_pars=header%3D99
The header parameter is optional, if not specified the primary header
is returned.
Valid values for the header parameter are numbers between 0 and 99. If
numbers are specified which are either outside the range or bigger than
the number of headers (including the primary) the primary header will be
returned. Headers are counted from 0 starting with the primary header. 99
is a special value as it returns all headers concatenated in a single file.
If 'xml=vo' is specified headers are returned using a slightly modified
VOTable (XML) format. If 'xml=xfits' is specified headers are returned
using the XFits (XML) format. struct=1 returns the structure of the FITS
file. tsv=1 returns the headers in a tab separated format suitable for
direct ingest into the header repository.
"""
logger.debug("Entering ngasExtractFitsHdrDppi() ...")
statusObj = ngamsDppiStatus.ngamsDppiStatus()
if (reqPropsObj.hasHttpPar("processing_pars")):
pars = ngamsPlugInApi.\
parseRawPlugInPars(reqPropsObj.getHttpPar("processing_pars"))
else:
# default is to extract the primary header
pars = {'header': 0}
logger.debug("ngasExtractFitsHdrDppi: %s %r", filename, pars)
PARS = set(['header', 'xml', 'skey', 'struct', 'tsv', 'check'])
# initial settings for printhead
xtract = 0
parse = 0
xmlVals = ['xfits', 'vo']
xmlfl = ''
skeyfl = 0
skey = 'END'
show = 0
struct = 0
tsv = 0
check = 0
mergefl = 0
hfl = 0
mode = 1
result = ''
err = ''
ext = 'hdr'
if pars.has_key('header'):
# extract a certain header: if value == 99 all headers are extracted,
# for any other value that header is extracted. headers are
# counted from 0
hfl = 1
struct = 0
show = pars['header']
try:
head = int(show)
except:
err = "ngasExtractFitsHdrDppi: Invalid type for header " +\
"parameter. Should be int"
if head < 0 or head > 99:
err = "ngasExtractFitsHdrDppi: Invalid value specified for " +\
"header parameter."
if pars.has_key('xml'):
# if this key exists we do a conversion to XFits XML.
struct = 0
if pars['xml'] in xmlVals:
xmlfl = pars['xml']
else:
err = "ngasExtractFitsHdrDppi: Invalid value for xml " +\
"parameter. Should be 'vo|xfits': "+ pars['xml']
ext = 'xml'
if pars.has_key('skey'):
# extract just one keyword. CAUTION: No checking done!
skey = pars['skey'].strip()
skeyfl = 1
keyParts = skey.split()
ext = 'txt'
head = int(head)
if head < 0: head = 0
if ((not re.match('[a-zA-Z]', skey[0]))
or (len(keyParts) > 1 and keyParts[0] != 'HIERARCH')
or (len(keyParts[0]) > 8)):
err = "ngasExtractFitsHdrDppi: Invalid value for skey " +\
"parameter specified. Must be a valid FITS keyword:",\
skey
if pars.has_key('struct'):
# return only the structure of the FITS file. Value of the
# parameter is ignored
head = -99
struct = 1
ext = 'txt'
if pars.has_key('tsv'):
# extract header in tsv format. Parameter value is ignored
struct = 1
tsv = 1
ext = 'txt'
head = int(head)
if head < 0: head = 0
if pars.has_key('check'):
# head structure and calculate the checksum of the data part.
head = -99
struct = 1
check = 1
# printhead supports a list of files, but here we only use one
fils = [filename]
base = os.path.basename(filename)
pos = base.rfind('.fits')
file_id = base[:pos]
if len(set(pars) - PARS) != 0: # detect unsupported parameters
xpars = set(pars) - PARS
err = "ngasExtractFitsHdrDppi: Unsupported option(s): %s\n" + \
"Valid options are:\n" + \
"header=<number> where number is an integer between 0 " +\
"and max(extension)-1 or 99 (for all)\n" +\
"xml=<format>, where format is [vo|xfits]\n" +\
"struct=1\n" +\
"skey=<FITS keyword>, should be a valid keyword, crude " +\
"checking is done\n" + \
"tsv=1\n\n" + \
"combinations are allowed and should be separated by " +\
"a comma character.\n"
err = err % xpars
ext = 'txt'
file_id = 'error'
result = '' # initialize result string
if err != '':
result = err
if result == '':
for f in fils:
cmd = constructCommand(f, head, struct, skey, tsv, xmlfl, mode,
check)
logger.debug('Executing command: %s', cmd)
stat, result, _ = execCmd(cmd)
if stat != 0:
errMsg = "Processing of header for file %s failed: %s" % (
filename, result)
raise Exception(errMsg)
resFilename = file_id + "." + ext
try:
# mime-type guessing does not work sometimes, we force it in that case.
mimeType = ngamsPlugInApi.determineMimeType(srvObj.getCfg(),
resFilename)
except:
if ext == 'xml':
mimeType = 'text/xml'
else:
mimeType = 'text/ascii'
resObj = ngamsDppiStatus.ngamsDppiResult(NGAMS_PROC_DATA, mimeType, result,
resFilename, '')
statusObj.addResult(resObj)
logger.debug("Leaving ngasExtractFitsHdrDppi() ...")
return statusObj
def ngamsExtractAlmaMultipart(srvObj, reqPropsObj, file_id):
"""
This DPPI extracts one part of a multipart/related message
requested from the ALMA Archive.
srvObj: Reference to instance of the NG/AMS Server
class (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
file_id: Name of file to process (string).
Returns: DPPI return status object (ngamsDppiStatus).
Side effect: This DPPI works directly on the archived file, since it is
read-only access.
SPECIFIC DOCUMENTATION:
This DPPI extracts one part of an ALMA multipart related message and
returns a complete, self-consistent message containing a XML header
and the requested cid. This version deals with VOTable headers and
returns a new VOTable header containing only the RESOURCE element
describing the requested (by cid) part. If no XML header can be found
or if the XML header is not a VOTable header this plugin returns just the
requested part, without any header.
Example URL (single line):
http://ngasdev1:7777/RETRIEVE
?file_id=X01/X7/X42&
processing=ngamsAlmaMultipart&
processing_pars='cid=<cid>'
"""
statusObj = ngamsDppiStatus.ngamsDppiStatus()
cpart = 1
pars = 0 # initialize pars
if (reqPropsObj.hasHttpPar("processing_pars")):
pars = ngamsPlugInApi.parseRawPlugInPars(\
reqPropsObj.getHttpPar("processing_pars"))
if pars and not pars.has_key('cid'):
ext = '.cid'
cpart = 1 # first part only by default
elif pars and pars.has_key(
'cid'): # if processing_par 'cid' exists check its contents
pass
else:
pass
resFilename = file_id + ext
try:
mimeType = ngamsPlugInApi.determineMimeType(srvObj.getCfg(),\
resFilename)
except:
pass
if ext == '.xml':
mimeType = 'text/xml'
else:
mimeType = 'multipart/related'
resObj = ngamsDppiStatus.ngamsDppiResult(NGAMS_PROC_DATA, mimeType, head,
resFilename, '')
statusObj.addResult(resObj)
return statusObj
def ngamsNgLogPlugIn(srvObj, reqPropsObj):
"""
Data Archiving Plug-In to handle archiving of NG/AMS (OLAS style) log
files.
srvObj: Reference to NG/AMS Server Object (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: Standard NG/AMS Data Archiving Plug-In Status
as generated by: ngamsPlugInApi.genDapiSuccessStat()
(ngamsDapiStatus).
"""
# For now the exception handling is pretty basic:
# If something goes wrong during the handling it is tried to
# move the temporary file to the Bad Files Area of the disk.
logger.debug("Plug-In handling data for file: %s",
os.path.basename(reqPropsObj.getFileUri()))
diskInfo = reqPropsObj.getTargDiskInfo()
stagingFilename = reqPropsObj.getStagingFilename()
ext = os.path.splitext(stagingFilename)[1][1:]
# Now, build final filename. We do that by taking the date of
# the first log entry.
#
# 2001-07-09T14:37:59.563 [INFO] Logging properties defined ...
#
# Alternatively the first entry could be of the form, e.g.:
#
# 2003-12-29T09:21:57.608 [INFO] LOG-ROTATE: 1072689717 - \
# SYSTEM-ID: ngamsArchiveClient@ngasdev2
#
# In the former case the Log ID is equal to the NGAS ID. In the latter
# case, the Log ID is equal to the System ID.
#
# The final filename is built as follows: <Log ID>.<date>.<ext>
#
# The file_id is: <Log ID>.<date>
fo = open(stagingFilename, "r")
firstLine = fo.readline()
fo.close()
try:
# Compress the log file.
uncomprSize = ngamsPlugInApi.getFileSize(stagingFilename)
compression = "gzip"
logger.debug("Compressing file using: %s ...", compression)
exitCode, stdOut = ngamsPlugInApi.execCmd("%s %s" %\
(compression,
stagingFilename))
if (exitCode != 0):
errMsg = "ngamsNgLogPlugIn: Problems during archiving! " +\
"Compressing the file failed"
raise Exception(errMsg)
stagingFilename = stagingFilename + ".gz"
# Remember to update the Temporary Filename in the Request
# Properties Object.
reqPropsObj.setStagingFilename(stagingFilename)
logger.debug("Log file compressed")
# Parse first line of the log file.
timeStamp = firstLine.split(" ")[0]
date = timeStamp.split("T")[0]
sysIdIdx = firstLine.find("SYSTEM-ID")
if (sysIdIdx != -1):
logId = firstLine[sysIdIdx + len("SYSTEM-ID:"):].strip().\
split(" ")[0]
else:
logId = srvObj.getHostId()
fileId = logId + "." + timeStamp
fileVersion, relPath, relFilename,\
complFilename, fileExists =\
ngamsPlugInApi.genFileInfo(srvObj.getDb(),
srvObj.getCfg(),
reqPropsObj, diskInfo,
stagingFilename, fileId,
fileId, [date])
# Generate status.
logger.debug("Generating status ...")
fformat = ngamsPlugInApi.determineMimeType(srvObj.getCfg(),
stagingFilename)
logger.debug("Determining file size ...")
fileSize = ngamsPlugInApi.getFileSize(stagingFilename)
return ngamsPlugInApi.genDapiSuccessStat(
diskInfo.getDiskId(), relFilename, fileId, fileVersion,
fformat, fileSize, uncomprSize, compression, relPath,
diskInfo.getSlotId(), fileExists, complFilename)
except Exception:
raise Exception("ngamsNgLogPlugIn: Error handling log file: " +\
stagingFilename + ". Rejecting.")
def ngamsGeneric(srvObj, reqPropsObj):
"""
Data Archiving Plug-In to handle archiving of SDM multipart related
message files containing ALMA UIDs in the Content-Location mime parameter
or any other kind of file
srvObj: Reference to NG/AMS Server Object (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: Standard NG/AMS Data Archiving Plug-In Status
as generated by: ngamsPlugInApi.genDapiSuccessStat()
(ngamsDapiStatus).
"""
logger.debug("Mirroring plug-in handling data for file: %s",
os.path.basename(reqPropsObj.getFileUri()))
# Create the file
diskInfo = reqPropsObj.getTargDiskInfo()
stagingFilename = reqPropsObj.getStagingFilename()
ext = os.path.splitext(stagingFilename)[1][1:]
# reqPropsObj format: /MIRRARCHIVE?mime_type=application/x-tar&filename=...
if (reqPropsObj.getMimeType()):
format = reqPropsObj.getMimeType()
else:
errMsg = "mime_type not specified in MIRRARCHIVE request"
raise Exception, errMsg
# File Uri format: http://ngasbe03.aiv.alma.cl:7777/RETRIEVE?disk_id=59622720f79296473f6106c15e5c2240&host_id=ngasbe03:7777&quick_location=1&file_version=1&file_id=backup.2011-02-02T22:01:59.tar
# Get file id
fileVersion = reqPropsObj.fileinfo['fileVersion']
fileId = reqPropsObj.fileinfo['fileId']
# Specific treatment depending on the mime type
if ((format.find("multipart") >= 0) or (format.find("multialma") >= 0)):
logger.debug("applying plug-in specific treatment")
fo = open(stagingFilename, "r")
try:
(fileId, finalName, format) = specificTreatment(fo)
finally:
fo.close()
else:
finalName = fileId
logger.debug(
"File with URI %s is being handled by ngamsGeneric: format=%s file_id=%s file_version=%s finalName=%s",
reqPropsObj.getFileUri(), format, fileId, fileVersion, finalName)
try:
# Compression parameters
uncomprSize = ngamsPlugInApi.getFileSize(stagingFilename)
compression = ""
# File name and paths
date = toiso8601(fmt=FMT_DATE_ONLY)
relPath, relFilename, complFilename, fileExists = genFileInfo(
srvObj.getDb(), srvObj.getCfg(), reqPropsObj, diskInfo,
stagingFilename, fileId, fileVersion, finalName, [date])
# Make sure the format is defined
if not format:
format = ngamsPlugInApi.determineMimeType(srvObj.getCfg(),
stagingFilename)
# FileSize
fileSize = ngamsPlugInApi.getFileSize(stagingFilename)
# Return resDapi object
return ngamsPlugInApi.genDapiSuccessStat(
diskInfo.getDiskId(), relFilename, fileId, fileVersion,
format, fileSize, uncomprSize, compression, relPath,
diskInfo.getSlotId(), fileExists, complFilename)
except Exception, e:
err = "Problem processing file in stagging area: " + str(e)
errMsg = genLog("NGAMS_ER_DAPI_BAD_FILE",
[stagingFilename, _PLUGIN_ID, err])
raise Exception, errMsg
