def ngams_generic(ngams_server, request_properties):
"""
Data Archiving Plug-In to handle archiving of SDM multipart related message files containing ALMA UIDs in the
Content-Location mime parameter or any other kind of file
:param ngams_server: Reference to NG/AMS Server Object (ngamsServer)
:param request_properties: NG/AMS request properties object (ngamsReqProps)
:return: Standard NG/AMS Data Archiving Plug-In Status as generated by: ngamsPlugInApi.genDapiSuccessStat()
(ngamsDapiStatus)
"""
logger.info("Mirroring plug-in handling data for file: %s", os.path.basename(request_properties.getFileUri()))
# Create staging file
disk_info = request_properties.getTargDiskInfo()
staging_filename = request_properties.getStagingFilename()
# request_properties format: /MIRRARCHIVE?mime_type=application/x-tar&filename=...
file_format = request_properties.getMimeType()
if not file_format:
raise Exception("mime_type parameter not specified in MIRRARCHIVE request")
# Example of file URI format:
# http://ngas01.org:7777/RETRIEVE?disk_id=59622720f79296473f6106c15e5c2240&host_id=ngas01:7777&quick_location=1&file_version=1&file_id=backup.2011-02-02T22:01:59.tar
file_id = request_properties.fileinfo["fileId"]
file_version = request_properties.fileinfo["fileVersion"]
# Specific treatment depending on the mime-type
if file_format.find("multipart") >= 0 or file_format.find("multialma") >= 0:
logger.debug("Mirroring plug-in applying specific treatment for multipart/multialma mime file")
file_id, final_filename, file_format = specific_treatment(staging_filename)
else:
final_filename = file_id
logger.debug("Mirroring plug-in processing request for file with URI %s, file_format=%s, file_id=%s, "
"file_version=%s, final_filename=%s", request_properties.getFileUri(), file_format, file_id,
file_version, final_filename)
try:
# Compression parameters
uncompressed_size = ngamsPlugInApi.getFileSize(staging_filename)
compression = ""
today = ngamsCore.toiso8601(fmt=ngamsCore.FMT_DATE_ONLY)
relative_path, relative_filename, complete_filename, file_exists = \
generate_file_info(ngams_server.getCfg(), disk_info, staging_filename, file_version, final_filename, [today])
# Make sure the format is defined
if not file_format:
file_format = ngamsPlugInApi.determineMimeType(ngams_server.getCfg(), staging_filename)
file_size = ngamsPlugInApi.getFileSize(staging_filename)
return ngamsPlugInApi.genDapiSuccessStat(disk_info.getDiskId(), relative_filename, file_id, file_version,
file_format, file_size, uncompressed_size, compression, relative_path,
disk_info.getSlotId(), file_exists, complete_filename)
except Exception as e:
raise Exception(genLog("NGAMS_ER_DAPI_BAD_FILE",
[staging_filename, PLUGIN_ID, "Problem processing file in staging area: " + str(e)]))
def ngamsSdmMultipart(ngams_server, request_properties):
"""
Data Archiving Plug-In to handle archiving of SDM multipart related message files containing ALMA UIDs in the
Content-Location mime parameter
:param ngams_server: Reference to NG/AMS Server Object (ngamsServer)
:param request_properties: NG/AMS request properties object (ngamsReqProps)
:return: Standard NG/AMS Data Archiving Plug-In Status as generated by: ngamsPlugInApi.genDapiSuccessStat()
(ngamsDapiStatus)
"""
# For now the exception handling is pretty basic:
# If something goes wrong during the handling it is tried to move the temporary file to the bad-files directory
logger.info("SDM multipart plug-in handling data for file: %s", os.path.basename(request_properties.getFileUri()))
disk_info = request_properties.getTargDiskInfo()
staging_filename = request_properties.getStagingFilename()
file_id, final_filename, file_format = specific_treatment(staging_filename)
if request_properties.hasHttpPar("file_id"):
file_id = request_properties.getHttpPar("file_id")
#if request_properties.hasHttpPar("file_version"):
# file_version = request_properties.getHttpPar("file_version")
logger.debug("SDM multipart plug-in processing request for file with URI %s, file_format=%s, file_id=%s, "
"final_filename=%s", request_properties.getFileUri(), file_format, file_id, final_filename)
try:
# Compression parameters
uncompressed_size = ngamsPlugInApi.getFileSize(staging_filename)
compression = ""
# Remember to update the temporary file name in the request properties object
request_properties.setStagingFilename(staging_filename)
today = ngamsCore.toiso8601(fmt=ngamsCore.FMT_DATE_ONLY)
file_version, relative_path, relative_filename, complete_filename, file_exists = \
ngamsPlugInApi.genFileInfo(ngams_server.getDb(), ngams_server.getCfg(), request_properties, disk_info,
staging_filename, file_id, final_filename, [today])
# Make sure the format is defined
if not file_format:
file_format = ngamsPlugInApi.determineMimeType(ngams_server.getCfg(), staging_filename)
file_size = ngamsPlugInApi.getFileSize(staging_filename)
return ngamsPlugInApi.genDapiSuccessStat(disk_info.getDiskId(), relative_filename, file_id, file_version,
file_format, file_size, uncompressed_size, compression, relative_path,
disk_info.getSlotId(), file_exists, complete_filename)
except Exception as e:
raise Exception(genLog("NGAMS_ER_DAPI_BAD_FILE",
[staging_filename, PLUGIN_ID, "Problem processing file in staging area: " + str(e)]))
def compress(reqPropsObj, parDic):
"""
Compress the file if required.
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
parDic: Dictionary with parameters for the DAPI. This is generated
with ngamsPlugInApi.parseDapiPlugInPars() (Dictionary).
Returns: Tupe containing uncompressed filesize, archived filesize
and the format (mime-type) of the resulting data file
(tuple).
"""
stFn = reqPropsObj.getStagingFilename()
uncomprSize = ngamsPlugInApi.getFileSize(stFn)
mime = reqPropsObj.getMimeType()
compression = parDic.get("compression")
if _compress_data(parDic):
logger.debug("Compressing file: %s using: %s", stFn, compression)
# Compress *and* calculate checksum on compressed stream
# The value crc_name depends on the server configuration and whether the
# user requested a different variant (see ngamsArchiveUtils)
gzip_name = '%s.gz' % stFn
crc_info = None
if 'crc_name' in reqPropsObj:
crc_info = ngamsFileUtils.get_checksum_info(
reqPropsObj['crc_name'])
compress_start = time.time()
with open(stFn, 'rb') as f:
crc = ngamsLib.gzip_compress(f,
gzip_name,
65536,
crc_info=crc_info)
compress_time = time.time() - compress_start
reqPropsObj.setStagingFilename(gzip_name)
rmFile(stFn)
mime = 'application/x-gfits'
compression = 'gzip --no-name'
logger.debug("File compressed: %s Time: %.3fs", gzip_name,
compress_time)
else:
compression = ''
crc = None
archFileSize = ngamsPlugInApi.getFileSize(reqPropsObj.getStagingFilename())
return uncomprSize, archFileSize, mime, compression, crc
def votable_catalogue_plugin(srv, request):
mime = request.getMimeType()
parDic = ngamsPlugInApi.parseDapiPlugInPars(srv.cfg, mime)
options = ('host', 'user', 'password', 'dbname')
for option in options:
if option not in parDic:
raise Exception(
'%s option not specified for votable_catalogue_plugin, please provide one'
% option)
host, user, password, dbname = [parDic[option] for option in options]
dsn = 'host=%s user=%s password=%s dbname=%s' % (host, user, password,
dbname)
stageFilename = request.getStagingFilename()
uncomprSize = ngamsPlugInApi.getFileSize(stageFilename)
file_id = os.path.basename(request.getFileUri())
diskInfo = request.getTargDiskInfo()
fileVersion, relPath, relFilename, complFilename, fileExists = \
ngamsPlugInApi.genFileInfo(srv.db, srv.cfg, request, diskInfo,
request.getStagingFilename(), file_id, file_id,
[], [])
insert_catalogue_dataproduct(dsn, file_id)
return ngamsPlugInApi.genDapiSuccessStat(diskInfo.getDiskId(), relFilename,
file_id, fileVersion, mime,
uncomprSize,
uncomprSize, '', relPath,
diskInfo.getSlotId(), fileExists,
complFilename)
def ngamsRegisterGenericPlugIn(server_object, request_object, param_dict):
"""
Generic registration plug-in to handle registration of files
:param server_object: Reference to NG/AMS Server Object (ngamsServer)
:param request_object: NG/AMS request properties object (ngamsReqProps)
:param param_dict: Parameter dictionary
:return: Standard NG/AMS Data Archiving Plug-In Status as generated by ngamsPlugInApi.genDapiSuccessStat()
(ngamsDapiStatus)
"""
logger.info("Register generic plug-in registering file with URI: %s", request_object.getFileUri())
disk_info = request_object.getTargDiskInfo()
mime_type = request_object.getMimeType()
stage_file = request_object.getStagingFilename()
file_id = os.path.basename(request_object.getFileUri())
if mime_type == ngamsCore.NGAMS_UNKNOWN_MT:
error_message = ngamsCore.genLog("NGAMS_ER_UNKNOWN_MIME_TYPE1", [file_id])
raise Exception(error_message)
file_size = ngamsPlugInApi.getFileSize(stage_file)
compression = ""
uncompressed_size = file_size
# Get various information about the file being handled
file_version, relative_path, relative_filename, complete_filename, file_exists = \
ngamsPlugInApi.genFileInfoReg(server_object.getDb(), server_object.getCfg(), request_object, disk_info,
stage_file, file_id)
logger.info("Register generic plug-in finished processing file with URI %s: file_id=%s, file_version=%s, format=%s, file_size=%s",
request_object.getFileUri(), file_id, file_version, mime_type, file_size)
return ngamsPlugInApi.genRegPiSuccessStat(disk_info.getDiskId(), relative_filename, file_id, file_version,
mime_type, file_size, uncompressed_size, compression, relative_path,
disk_info.getSlotId(), file_exists, complete_filename)
def generic_register_plugin(srv, req, _params):
'''A simple, generic register plug-in entry point used by one of our tests'''
diskInfo = req.getTargDiskInfo()
stageFile = req.getStagingFilename()
mime_type = req.getMimeType()
file_id = os.path.basename(req.getFileUri())
compresion = ""
fileVersion, relPath, relFilename, complFilename, fileExists = \
ngamsPlugInApi.genFileInfoReg(srv.db, srv.cfg, req, diskInfo,
stageFile, file_id)
uncomprSize = ngamsPlugInApi.getFileSize(stageFile)
return ngamsPlugInApi.genRegPiSuccessStat(
diskInfo.getDiskId(), relFilename, file_id, fileVersion,
mime_type, uncomprSize, uncomprSize, compresion, relPath,
diskInfo.getSlotId(), fileExists, complFilename)
def checkFitsFileSize(filename):
"""
Check if the size of the FITS file is a multiple of 2880. If this
is not the case, we through an exception.
filename: FITS file to check (string).
Returns: Void.
"""
if filename.lower().endswith('.fits'):
size = ngamsPlugInApi.getFileSize(filename)
if size % 2880 != 0:
errMsg = ("The size of the FITS file issued "
"is not a multiple of 2880 (size: %d)! Rejecting file!")
errMsg = genLog(
"NGAMS_ER_DAPI_BAD_FILE",
[os.path.basename(filename), "ngamsFitsPlugIn", errMsg % size])
raise Exception, errMsg
def ngasTarBallPlugIn(srvObj, reqPropsObj):
"""
Data Archiving Plug-In to handle archiving of tarballs.
srvObj: Reference to NG/AMS Server Object (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: Standard NG/AMS Data Archiving Plug-In Status
as generated by: ngamsPlugInApi.genDapiSuccessStat()
(ngamsDapiStatus).
"""
stagingFilename = reqPropsObj.getStagingFilename()
logger.info("Plug-In handling data for file with URI: %s",
os.path.basename(reqPropsObj.getFileUri()))
diskInfo = reqPropsObj.getTargDiskInfo()
# Check file.
checkTarball(stagingFilename)
# Get various information about the file being handled.
fileId = os.path.basename(reqPropsObj.getFileUri())
timestamp = fileId[fileId.find(".") + 1:]
obsDay = tomjd(fromiso8601(timestamp)) - 0.5
dateDirName = toiso8601(obsDay, fmt=FMT_DATE_ONLY)
baseFilename = fileId[0:-4]
fileVersion, relPath, relFilename,\
complFilename, fileExists =\
ngamsPlugInApi.genFileInfo(srvObj.getDb(), srvObj.getCfg(),
reqPropsObj, diskInfo,
stagingFilename, fileId,
baseFilename, [dateDirName], [])
# Generate status.
logger.debug("Generating status ...")
format = "application/x-tar"
fileSize = ngamsPlugInApi.getFileSize(stagingFilename)
logger.debug("DAPI finished processing of file")
return ngamsPlugInApi.genDapiSuccessStat(diskInfo.getDiskId(), relFilename,
fileId, fileVersion, format,
fileSize, fileSize,
"NONE", relPath,
diskInfo.getSlotId(), fileExists,
complFilename)
def ngamsAlmaMultipart(srvObj,
reqPropsObj):
"""
Data Archiving Plug-In to handle archiving of ALMA multipart related
message files containing ALMA UIDs.
srvObj: Reference to NG/AMS Server Object (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: Standard NG/AMS Data Archiving Plug-In Status
as generated by: ngamsPlugInApi.genDapiSuccessStat()
(ngamsDapiStatus).
"""
# For now the exception handling is pretty basic:
# If something goes wrong during the handling it is tried to
# move the temporary file to the Bad Files Area of the disk.
logger.info("Plug-In handling data for file: " +
os.path.basename(reqPropsObj.getFileUri()))
diskInfo = reqPropsObj.getTargDiskInfo()
stagingFilename = reqPropsObj.getStagingFilename()
ext = os.path.splitext(stagingFilename)[1][1:]
fo = open(stagingFilename, "r")
(fileId, finalName, format) = specificTreatment(fo)
fo.close()
try:
# Compress the file.
uncomprSize = ngamsPlugInApi.getFileSize(stagingFilename)
compression = ""
# info(2,"Compressing file using: %s ..." % compression)
# exitCode, stdOut = ngamsPlugInApi.execCmd("%s %s" %\
# (compression,
# stagingFilename))
# if (exitCode != 0):
# errMsg = _PLUGIN_ID+": Problems during archiving! " +\
# "Compressing the file failed"
# raise Exception, errMsg
# stagingFilename = stagingFilename + ".Z"
# Remember to update the Temporary Filename in the Request
# Properties Object.
reqPropsObj.setStagingFilename(stagingFilename)
# info(2,"File compressed")
# ToDo: Handling of non-existing fileId
# if (fileId == -1):
# fileId = ngamsPlugInApi.genNgasId(srvObj.getCfg())
date = toiso8601(local=True, fmt=FMT_DATE_ONLY)
fileVersion, relPath, relFilename,\
complFilename, fileExists =\
ngamsPlugInApi.genFileInfo(srvObj.getDb(),
srvObj.getCfg(),
reqPropsObj, diskInfo,
stagingFilename, fileId,
finalName, [date])
# Generate status.
logger.debug("Generating status ...")
if not format:
format = ngamsPlugInApi.determineMimeType(srvObj.getCfg(),
stagingFilename)
fileSize = ngamsPlugInApi.getFileSize(stagingFilename)
return ngamsPlugInApi.genDapiSuccessStat(diskInfo.getDiskId(),
relFilename,
fileId, fileVersion, format,
fileSize, uncomprSize,
compression, relPath,
diskInfo.getSlotId(),
fileExists, complFilename)
except Exception as e:
errMsg = genLog("NGAMS_ER_DAPI_BAD_FILE", [os.path.basename(stagingFilename),
_PLUGIN_ID, str(e)])
raise Exception(errMsg)
def compressFile(srvObj, reqPropsObj, parDic):
"""
Compress the file if required.
srvObj: Reference to NG/AMS Server Object (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
parDic: Dictionary with parameters for the DAPI. This is generated
with ngamsPlugInApi.parseDapiPlugInPars() (Dictionary).
Returns: Tupe containing uncompressed filesize, archived filesize
and the format (mime-type) of the resulting data file and
the compression method (NONE if the file is not compressed),
finally, the extension added by the compression if any
(tuple).
"""
stFn = reqPropsObj.getStagingFilename()
# If a compression application is specified, apply this.
uncomprSize = ngamsPlugInApi.getFileSize(stFn)
comprExt = ""
if (parDic[COMPRESSION]):
logger.debug("Compressing file using: %s ...", parDic[COMPRESSION])
compCmd = "%s %s" % (parDic[COMPRESSION], stFn)
compress_start = time.time()
logger.debug("Compressing file with command: %s", compCmd)
exitCode, stdOut = ngamsPlugInApi.execCmd(compCmd)
#if (exitCode != 0):
# msg ="Problems during archiving! Compressing the file failed. " +\
# "Error: %s" % str(stdOut).replace("/n", " ")
# raise Exception, msg
# If the compression fails, assume that it is because the file is not
# compressible (although it could also be due to lack of disk space).
if (exitCode == 0):
if (parDic[COMPRESSION_EXT]):
stFn = stFn + "." + parDic[COMPRESSION_EXT]
comprExt = parDic[COMPRESSION_EXT]
# Remember to update Staging Filename in the Request Properties
# Object.
reqPropsObj.setStagingFilename(stFn)
# Handle mime-type
if (parDic[TARG_MIME_TYPE]):
format = parDic[TARG_MIME_TYPE]
else:
format = ngamsPlugInApi.determineMimeType(
srvObj.getCfg(), stFn)
compression = parDic[COMPRESSION]
logger.debug("File compressed. Time: %.3fs",
time.time() - compress_start)
else:
# Carry on with the original file. We take the original mime-type
# as the target mime-type.
format = reqPropsObj.getMimeType()
compression = NO_COMPRESSION
else:
# Handle mime-type
if (parDic[TARG_MIME_TYPE]):
format = parDic[TARG_MIME_TYPE]
else:
format = reqPropsObj.getMimeType()
compression = NO_COMPRESSION
archFileSize = ngamsPlugInApi.getFileSize(reqPropsObj.getStagingFilename())
return uncomprSize, archFileSize, format, compression, comprExt
def ngamsMWAFitsPlugIn(srvObj, reqPropsObj):
logger.info("ngamsMWAFitsPlugIn handling data for file with URI: %s",
os.path.basename(reqPropsObj.getFileUri()))
mime = reqPropsObj.getMimeType()
parDic = ngamsPlugInApi.parseDapiPlugInPars(srvObj.getCfg(), mime)
with DBPool.db_lock:
if DBPool.db_pool is None:
try:
dbhost = parDic['dbhost']
dbname = parDic['dbname']
dbuser = parDic['dbuser']
dbpass = parDic['dbpassword']
except Exception as e:
raise Exception(
'Must specify parameters dbhost, dbname, dbuser and dbpassword'
)
maxconn = 8
try:
maxconn = parDic['dbmaxconn']
except:
pass
logger.info("ngamsMWAFitsPlugIn creating database pool")
DBPool.db_pool = psycopg2.pool.ThreadedConnectionPool(
minconn=1,
maxconn=maxconn,
host=dbhost,
user=dbuser,
database=dbname,
password=dbpass,
port=5432)
diskInfo = reqPropsObj.getTargDiskInfo()
stageFilename = reqPropsObj.getStagingFilename()
uncomprSize = ngamsPlugInApi.getFileSize(stageFilename)
fileName = os.path.basename(reqPropsObj.getFileUri())
dpId = fileName
obsid, obstime, box = split_file(fileName)
fileVersion, relPath, relFilename, complFilename, fileExists = \
ngamsPlugInApi.genFileInfo(srvObj.getDb(),
srvObj.getCfg(),
reqPropsObj,
diskInfo,
reqPropsObj.getStagingFilename(),
dpId,
dpId,
[],
[])
logger.info("ngamsMWAFitsPlugIn version: %s path: %s relfilename: %s" %
(fileVersion, relPath, relFilename))
sql = (
'INSERT INTO data_files (observation_num, filetype, size, filename, site_path, host)'
' VALUES (%s, %s, %s, %s, %s, %s)')
uri = 'http://mwangas/RETRIEVE?file_id=%s' % (fileName)
logger.info('Inserting: %s',
sql % (obsid, 8, uncomprSize, fileName, uri, box))
conn = None
try:
conn = DBPool.db_pool.getconn()
conn.set_isolation_level(
psycopg2.extensions.ISOLATION_LEVEL_READ_COMMITTED)
with conn.cursor() as cur:
try:
cur.execute(sql, [obsid, 8, uncomprSize, fileName, uri, box])
except Exception as e:
''' Do not want to raise an exception here because the sky data is more important than the metadata
i.e. we want to keep the sky data even if there is a database problem.
If there is an issue with an INSERT then go through and add it manually later (very unlikely to happen)'''
conn.rollback()
logger.error('Insert error: %s Message: %s',
sql % (obsid, 8, uncomprSize, fileName, uri, box),
str(e))
else:
conn.commit()
except Exception:
logger.exception('Insert error: %s',
sql % (obsid, 8, uncomprSize, fileName, uri, box))
finally:
if conn:
DBPool.db_pool.putconn(conn=conn)
return ngamsPlugInApi.genDapiSuccessStat(diskInfo.getDiskId(), relFilename,
dpId, fileVersion, mime,
uncomprSize,
uncomprSize, '', relPath,
diskInfo.getSlotId(), fileExists,
complFilename)
def ngamsFitsRegPlugIn(srvObj, reqPropsObj, parDic):
"""
Data Registration Plug-In to handle registration of FITS files.
srvObj: Reference to NG/AMS Server Object (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: Standard NG/AMS Data Archiving Plug-In Status as generated
by: ngamsPlugInApi.genDapiSuccessStat() (ngamsDapiStatus).
"""
logger.info("Plug-In registering file with URI: %s", reqPropsObj.getFileUri())
diskInfo = reqPropsObj.getTargDiskInfo()
stageFile = reqPropsObj.getStagingFilename()
# If the file is already compressed, we have to decompress it.
procDir = ""
if ((stageFile.find(".Z") != -1) or (stageFile.find(".gz") != -1)):
workingFile, procDir = ngamsPlugInApi.prepProcFile(srvObj.getCfg(),
stageFile)
ngamsPlugInApi.execCmd("gunzip " + workingFile)
if (workingFile.find(".Z") != -1):
workingFile = workingFile[:-2]
else:
workingFile = workingFile[:-3]
else:
workingFile = stageFile
# Check file (size + checksum).
ngamsFitsPlugIn.checkFitsFileSize(workingFile)
#ngamsFitsPlugIn.c_heckChecksum(parDic, workingFile)
if 'skip_checksum' not in parDic:
ngamsFitsPlugIn.checkFitsChecksum(reqPropsObj, workingFile)
# Get various information about the file being handled.
arcFile, dpId, dateDirName = ngamsFitsPlugIn.getDpIdInfo(workingFile)
fileVersion, relPath, relFilename,\
complFilename, fileExists =\
ngamsPlugInApi.genFileInfoReg(srvObj.getDb(), srvObj.getCfg(),
reqPropsObj, diskInfo,
stageFile, dpId)
# Generate status.
logger.debug("Generating status ...")
fileSize = ngamsPlugInApi.getFileSize(stageFile)
if (stageFile.find(".Z") != -1):
format = "application/x-cfits"
compresion = "compress"
elif (stageFile.find(".gz") != -1):
format = "application/x-gfits"
compresion = "gzip"
else:
format = "image/x-fits"
compresion = ""
uncomprSize = ngamsPlugInApi.getFileSize(workingFile)
# Delete the processing directory (would be done later by the
# Janitor Thread, but it is better to clean up explicitly).
if (procDir): rmFile(procDir)
logger.debug("Register Plug-In finished processing of file")
return ngamsPlugInApi.genRegPiSuccessStat(diskInfo.getDiskId(),relFilename,
dpId, fileVersion, format,
fileSize, uncomprSize,compresion,
relPath, diskInfo.getSlotId(),
fileExists, complFilename)
def handleCmd(srvObj, reqPropsObj, httpRef):
"""
Handle the CARCHIVE Command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: (fileId, filePath) tuple.
"""
T = TRACE()
# Is this NG/AMS permitted to handle Archive Requests?
logger.debug("Is this NG/AMS permitted to handle Archive Requests?")
if (not srvObj.getCfg().getAllowArchiveReq()):
errMsg = genLog("NGAMS_ER_ILL_REQ", ["Archive"])
raise Exception(errMsg)
srvObj.checkSetState("Archive Request", [NGAMS_ONLINE_STATE],
[NGAMS_IDLE_SUBSTATE, NGAMS_BUSY_SUBSTATE],
NGAMS_ONLINE_STATE,
NGAMS_BUSY_SUBSTATE,
updateDb=False)
if httpRef.command != 'POST':
raise Exception("Only POST allowed for CARCHIVE")
# Get mime-type (try to guess if not provided as an HTTP parameter).
logger.debug(
"Get mime-type (try to guess if not provided as an HTTP parameter).")
if (reqPropsObj.getMimeType() == ""):
mimeType = ngamsHighLevelLib.\
determineMimeType(srvObj.getCfg(), reqPropsObj.getFileUri())
reqPropsObj.setMimeType(mimeType)
else:
mimeType = reqPropsObj.getMimeType()
# Determine the target volume, ignoring the stream concept.
logger.debug("Determine the target volume, ignoring the stream concept.")
targDiskInfo = ngamsArchiveUtils._random_target_volume(srvObj)
if (targDiskInfo == None):
errMsg = "No disk volumes are available for ingesting any files."
raise Exception(errMsg)
reqPropsObj.setTargDiskInfo(targDiskInfo)
# Generate staging filename.
logger.debug("Generate staging filename from URI: %s",
reqPropsObj.getFileUri())
if (reqPropsObj.getFileUri().find("file_id=") >= 0):
file_id = reqPropsObj.getFileUri().split("file_id=")[1]
baseName = os.path.basename(file_id)
else:
baseName = os.path.basename(reqPropsObj.getFileUri())
stgFilename = os.path.join("/", targDiskInfo.getMountPoint(),
NGAMS_STAGING_DIR,
genUniqueId() + "___" + baseName)
logger.debug("Staging filename is: %s", stgFilename)
reqPropsObj.setStagingFilename(stgFilename)
# Retrieve file contents (from URL, archive pull, or by storing the body
# of the HTTP request, archive push).
stagingInfo = saveInStagingFile(srvObj.getCfg(), reqPropsObj, httpRef,
stgFilename, targDiskInfo)
ioTime = stagingInfo[0]
rootContainer = stagingInfo[1]
fileDataList = stagingInfo[2]
ingestRate = stagingInfo[3]
reqPropsObj.incIoTime(ioTime)
createContainers(rootContainer, None, srvObj)
parDic = {}
ngamsGenDapi.handlePars(reqPropsObj, parDic)
diskInfo = reqPropsObj.getTargDiskInfo()
# Generate file information.
logger.debug("Generate file information")
dateDir = toiso8601(fmt=FMT_DATE_ONLY)
resDapiList = []
containerSizes = {}
for item in fileDataList:
container = item[0]
filepath = item[1]
crc = item[2]
containerId = str(container.getContainerId())
basename = os.path.basename(filepath)
fileId = basename
fileVersion, relPath, relFilename,\
complFilename, fileExists =\
ngamsPlugInApi.genFileInfo(srvObj.getDb(),
srvObj.getCfg(),
reqPropsObj, diskInfo,
filepath,
fileId,
basename, [dateDir])
complFilename, relFilename = ngamsGenDapi.checkForDblExt(
complFilename, relFilename)
# Keep track of the total size of the container
uncomprSize = ngamsPlugInApi.getFileSize(filepath)
if containerId not in containerSizes:
containerSizes[containerId] = 0
containerSizes[containerId] += uncomprSize
mimeType = reqPropsObj.getMimeType()
compression = "NONE"
archFileSize = ngamsPlugInApi.getFileSize(filepath)
resDapi = ngamsPlugInApi.genDapiSuccessStat(
diskInfo.getDiskId(), relFilename, fileId, fileVersion, mimeType,
archFileSize, uncomprSize, compression, relPath,
diskInfo.getSlotId(), fileExists, complFilename)
# Move file to final destination.
logger.debug("Moving file to final destination")
ioTime = mvFile(filepath, resDapi.getCompleteFilename())
reqPropsObj.incIoTime(ioTime)
# Get crc info
checksumPlugIn = "StreamCrc32"
checksum = str(crc)
# Get source file version
# e.g.: http://ngas03.hq.eso.org:7778/RETRIEVE?file_version=1&file_id=X90/X962a4/X1
logger.debug("Get file version")
file_version = resDapi.getFileVersion()
if reqPropsObj.getFileUri().count("file_version"):
file_version = int(
(reqPropsObj.getFileUri().split("file_version=")[1]
).split("&")[0])
# Check/generate remaining file info + update in DB.
logger.debug("Creating db entry")
creDate = getFileCreationTime(resDapi.getCompleteFilename())
fileInfo = ngamsFileInfo.ngamsFileInfo().\
setDiskId(resDapi.getDiskId()).\
setFilename(resDapi.getRelFilename()).\
setFileId(resDapi.getFileId()).\
setFileVersion(file_version).\
setFormat(resDapi.getFormat()).\
setFileSize(resDapi.getFileSize()).\
setUncompressedFileSize(resDapi.getUncomprSize()).\
setCompression(resDapi.getCompression()).\
setIngestionDate(time.time()).\
setChecksum(checksum).setChecksumPlugIn(checksumPlugIn).\
setFileStatus(NGAMS_FILE_STATUS_OK).\
setCreationDate(creDate).\
setIoTime(reqPropsObj.getIoTime())
fileInfo.write(srvObj.getHostId(), srvObj.getDb())
# Add the file to the container
srvObj.getDb().addFileToContainer(containerId, resDapi.getFileId(),
True)
# Update the container sizes
for contSizeInfo in containerSizes.items():
srvObj.getDb().setContainerSize(contSizeInfo[0], contSizeInfo[1])
# Inform the caching service about the new file.
logger.debug("Inform the caching service about the new file.")
if (srvObj.getCachingActive()):
diskId = resDapi.getDiskId()
fileId = resDapi.getFileId()
fileVersion = file_version
filename = resDapi.getRelFilename()
ngamsCacheControlThread.addEntryNewFilesDbm(
srvObj, diskId, fileId, fileVersion, filename)
# Update disk info in NGAS Disks.
logger.debug("Update disk info in NGAS Disks.")
srvObj.getDb().updateDiskInfo(resDapi.getFileSize(),
resDapi.getDiskId())
resDapiList.append(resDapi)
# Check if the disk is completed.
# We use an approximate extimate for the remaning disk space to avoid
# to read the DB.
logger.debug("Check available space in disk")
availSpace = getDiskSpaceAvail(targDiskInfo.getMountPoint(), smart=False)
if (availSpace < srvObj.getCfg().getFreeSpaceDiskChangeMb()):
targDiskInfo.setCompleted(1).setCompletionDate(time.time())
targDiskInfo.write(srvObj.getDb())
# Request after-math ...
srvObj.setSubState(NGAMS_IDLE_SUBSTATE)
msg = "Successfully handled Archive Pull Request for data file " +\
"with URI: " + reqPropsObj.getSafeFileUri()
logger.info(msg)
httpRef.send_ingest_status(msg, targDiskInfo)
for resDapi in resDapiList:
# Trigger Subscription Thread. This is a special version for MWA, in which we simply swapped MIRRARCHIVE and QARCHIVE
# [email protected]
logger.debug("triggering SubscriptionThread for file %s",
resDapi.getFileId())
srvObj.addSubscriptionInfo(
[(resDapi.getFileId(), resDapi.getFileVersion())], [])
srvObj.triggerSubscriptionThread()
def ngamsNgLogPlugIn(srvObj, reqPropsObj):
"""
Data Archiving Plug-In to handle archiving of NG/AMS (OLAS style) log
files.
srvObj: Reference to NG/AMS Server Object (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: Standard NG/AMS Data Archiving Plug-In Status
as generated by: ngamsPlugInApi.genDapiSuccessStat()
(ngamsDapiStatus).
"""
# For now the exception handling is pretty basic:
# If something goes wrong during the handling it is tried to
# move the temporary file to the Bad Files Area of the disk.
logger.debug("Plug-In handling data for file: %s",
os.path.basename(reqPropsObj.getFileUri()))
diskInfo = reqPropsObj.getTargDiskInfo()
stagingFilename = reqPropsObj.getStagingFilename()
ext = os.path.splitext(stagingFilename)[1][1:]
# Now, build final filename. We do that by taking the date of
# the first log entry.
#
# 2001-07-09T14:37:59.563 [INFO] Logging properties defined ...
#
# Alternatively the first entry could be of the form, e.g.:
#
# 2003-12-29T09:21:57.608 [INFO] LOG-ROTATE: 1072689717 - \
# SYSTEM-ID: ngamsArchiveClient@ngasdev2
#
# In the former case the Log ID is equal to the NGAS ID. In the latter
# case, the Log ID is equal to the System ID.
#
# The final filename is built as follows: <Log ID>.<date>.<ext>
#
# The file_id is: <Log ID>.<date>
fo = open(stagingFilename, "r")
firstLine = fo.readline()
fo.close()
try:
# Compress the log file.
uncomprSize = ngamsPlugInApi.getFileSize(stagingFilename)
compression = "gzip"
logger.debug("Compressing file using: %s ...", compression)
exitCode, stdOut = ngamsPlugInApi.execCmd("%s %s" %\
(compression,
stagingFilename))
if (exitCode != 0):
errMsg = "ngamsNgLogPlugIn: Problems during archiving! " +\
"Compressing the file failed"
raise Exception(errMsg)
stagingFilename = stagingFilename + ".gz"
# Remember to update the Temporary Filename in the Request
# Properties Object.
reqPropsObj.setStagingFilename(stagingFilename)
logger.debug("Log file compressed")
# Parse first line of the log file.
timeStamp = firstLine.split(" ")[0]
date = timeStamp.split("T")[0]
sysIdIdx = firstLine.find("SYSTEM-ID")
if (sysIdIdx != -1):
logId = firstLine[sysIdIdx + len("SYSTEM-ID:"):].strip().\
split(" ")[0]
else:
logId = srvObj.getHostId()
fileId = logId + "." + timeStamp
fileVersion, relPath, relFilename,\
complFilename, fileExists =\
ngamsPlugInApi.genFileInfo(srvObj.getDb(),
srvObj.getCfg(),
reqPropsObj, diskInfo,
stagingFilename, fileId,
fileId, [date])
# Generate status.
logger.debug("Generating status ...")
fformat = ngamsPlugInApi.determineMimeType(srvObj.getCfg(),
stagingFilename)
logger.debug("Determining file size ...")
fileSize = ngamsPlugInApi.getFileSize(stagingFilename)
return ngamsPlugInApi.genDapiSuccessStat(
diskInfo.getDiskId(), relFilename, fileId, fileVersion,
fformat, fileSize, uncomprSize, compression, relPath,
diskInfo.getSlotId(), fileExists, complFilename)
except Exception:
raise Exception("ngamsNgLogPlugIn: Error handling log file: " +\
stagingFilename + ". Rejecting.")
def ngamsGeneric(srvObj, reqPropsObj):
"""
Data Archiving Plug-In to handle archiving of SDM multipart related
message files containing ALMA UIDs in the Content-Location mime parameter
or any other kind of file
srvObj: Reference to NG/AMS Server Object (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: Standard NG/AMS Data Archiving Plug-In Status
as generated by: ngamsPlugInApi.genDapiSuccessStat()
(ngamsDapiStatus).
"""
logger.debug("Mirroring plug-in handling data for file: %s",
os.path.basename(reqPropsObj.getFileUri()))
# Create the file
diskInfo = reqPropsObj.getTargDiskInfo()
stagingFilename = reqPropsObj.getStagingFilename()
ext = os.path.splitext(stagingFilename)[1][1:]
# reqPropsObj format: /MIRRARCHIVE?mime_type=application/x-tar&filename=...
if (reqPropsObj.getMimeType()):
format = reqPropsObj.getMimeType()
else:
errMsg = "mime_type not specified in MIRRARCHIVE request"
raise Exception, errMsg
# File Uri format: http://ngasbe03.aiv.alma.cl:7777/RETRIEVE?disk_id=59622720f79296473f6106c15e5c2240&host_id=ngasbe03:7777&quick_location=1&file_version=1&file_id=backup.2011-02-02T22:01:59.tar
# Get file id
fileVersion = reqPropsObj.fileinfo['fileVersion']
fileId = reqPropsObj.fileinfo['fileId']
# Specific treatment depending on the mime type
if ((format.find("multipart") >= 0) or (format.find("multialma") >= 0)):
logger.debug("applying plug-in specific treatment")
fo = open(stagingFilename, "r")
try:
(fileId, finalName, format) = specificTreatment(fo)
finally:
fo.close()
else:
finalName = fileId
logger.debug(
"File with URI %s is being handled by ngamsGeneric: format=%s file_id=%s file_version=%s finalName=%s",
reqPropsObj.getFileUri(), format, fileId, fileVersion, finalName)
try:
# Compression parameters
uncomprSize = ngamsPlugInApi.getFileSize(stagingFilename)
compression = ""
# File name and paths
date = toiso8601(fmt=FMT_DATE_ONLY)
relPath, relFilename, complFilename, fileExists = genFileInfo(
srvObj.getDb(), srvObj.getCfg(), reqPropsObj, diskInfo,
stagingFilename, fileId, fileVersion, finalName, [date])
# Make sure the format is defined
if not format:
format = ngamsPlugInApi.determineMimeType(srvObj.getCfg(),
stagingFilename)
# FileSize
fileSize = ngamsPlugInApi.getFileSize(stagingFilename)
# Return resDapi object
return ngamsPlugInApi.genDapiSuccessStat(
diskInfo.getDiskId(), relFilename, fileId, fileVersion,
format, fileSize, uncomprSize, compression, relPath,
diskInfo.getSlotId(), fileExists, complFilename)
except Exception, e:
err = "Problem processing file in stagging area: " + str(e)
errMsg = genLog("NGAMS_ER_DAPI_BAD_FILE",
[stagingFilename, _PLUGIN_ID, err])
raise Exception, errMsg
