def _delFile(srvObj, path, hostId, execute):
"""
Delete (remove from disk) the file specified by the path on the given host.
srvObj: Reference to NG/AMS server class object (ngamsServer).
path: Path of file (string).
hostId: ID of host where file is stored (string).
Returns: Message with info about the execution (string).
"""
if (hostId and (hostId != srvObj.getHostId())):
raise Exception, "DISCARD Command can only be executed locally!"
# Handle the discard request locally.
ngasHostId = "%s:%d" % (getHostName(), srvObj.getCfg().getPortNo())
if (os.path.exists(path)):
if (not ngamsLib.fileRemovable(path)):
return genLog("NGAMS_ER_DISCARD_NOT_REM", [path, ngasHostId])
if (not execute):
msg = genLog("NGAMS_INFO_DISCARD_GRANTED", [path, ngasHostId])
else:
rmFile(path)
if (os.path.exists(path)):
return genLog("NGAMS_ER_DISCARD_FAILED",
[path, ngasHostId, "Not removable"])
else:
msg = genLog("NGAMS_INFO_DISCARD_OK", [path, ngasHostId])
else:
return genLog("NGAMS_ER_DISCARD_NOT_FOUND", [path, ngasHostId])
return msg
def ngamsRaiseEx_NGAMS_ER_DAPI_1(srvObj, reqPropsObj):
"""
Test DAPI, which raises an NGAMS_ER_DAPI Exception.
Returns: Void.
"""
raise Exception, genLog("NGAMS_ER_DAPI",
[genLog("NGAMS_ER_DB_COM", ["TEST EXCEPTION"])])
def handleCmd(srvObj, reqPropsObj, httpRef):
"""
Handle DISCARD Command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
if (reqPropsObj.hasHttpPar("help")):
global _help
return _help
if (not srvObj.getCfg().getAllowRemoveReq()):
errMsg = genLog("NGAMS_ER_ILL_REQ", ["Discard File"])
raise Exception(errMsg)
diskId = None
fileId = None
fileVersion = None
hostId = None
path = None
execute = 0
if (reqPropsObj.hasHttpPar("disk_id")):
diskId = reqPropsObj.getHttpPar("disk_id")
if (reqPropsObj.hasHttpPar("file_id")):
fileId = reqPropsObj.getHttpPar("file_id")
if (reqPropsObj.hasHttpPar("file_version")):
try:
fileVersion = int(reqPropsObj.getHttpPar("file_version"))
except ValueError:
raise Exception("Illegal value for File Version specified: " +\
str(fileVersion))
if (reqPropsObj.hasHttpPar("host_id")):
hostId = reqPropsObj.getHttpPar("host_id")
if (reqPropsObj.hasHttpPar("path")):
path = reqPropsObj.getHttpPar("path")
if (reqPropsObj.hasHttpPar("execute")):
try:
execute = int(reqPropsObj.getHttpPar("execute"))
except:
errMsg = genLog("NGAMS_ER_REQ_HANDLING", ["Must provide proper " +\
"value for parameter: execute (0|1)"])
raise Exception(errMsg)
msg = _discardFile(srvObj, diskId, fileId, fileVersion, hostId, path,
execute)
if msg.startswith("NGAMS_INFO_"):
status = NGAMS_SUCCESS
else:
status = NGAMS_FAILURE
httpRef.send_status(msg, status=status, code=NGAMS_HTTP_SUCCESS)
def specificTreatment(fo):
"""
Method contains the specific treatment of the file passed from NG/AMS.
fo: File object
Returns: (file_id, finalFileName, type);
The finalFileName is a string containing the name of the final
file without extension. type is the mime-type from the header.
"""
import rfc822, cgi, re
_EXT = '.msg'
filename = fo.name
uidTempl = re.compile(
"^[uU][iI][dD]://[aAbBcCzZxX][0-9,a-z,A-Z]+(/[xX][0-9,a-z,A-Z]+){2}(#\w{1,}|/\w{0,}){0,}$"
)
try:
message = rfc822.Message(fo)
type, tparams = cgi.parse_header(message["Content-Type"])
except Exception as e:
err = "Parsing of mime message failed: " + str(e)
errMsg = genLog("NGAMS_ER_DAPI_BAD_FILE",
[os.path.basename(filename), _PLUGIN_ID, err])
raise Exception(errMsg)
try:
almaUid = message["alma-uid"]
except:
try:
almaUid = message["Content-Location"]
except:
err = "Mandatory alma-uid or Content-Location parameter not found in mime header!"
errMsg = genLog("NGAMS_ER_DAPI_BAD_FILE",
[os.path.basename(filename), _PLUGIN_ID, err])
raise Exception(errMsg)
if not uidTempl.match(almaUid):
err = "Invalid alma-uid found in Content-Location: " + almaUid
errMsg = genLog("NGAMS_ER_DAPI_BAD_FILE",
[os.path.basename(filename), _PLUGIN_ID, err])
raise Exception(errMsg)
try:
almaUid = almaUid.split('//', 2)[1].split('#')[0]
if almaUid[-1] == '/': almaUid = almaUid[:-1]
fileId = almaUid
finalFileName = almaUid.replace('/', ':')
except Exception as e:
err = "Problem constructing final file name: " + str(e)
errMsg = genLog("NGAMS_ER_DAPI_BAD_FILE",
[os.path.basename(filename), _PLUGIN_ID, err])
raise Exception(errMsg)
return (fileId, finalFileName, type)
def _checkFileAccess(srvObj,
reqPropsObj,
httpRef,
fileId,
fileVersion=-1,
diskId=""):
"""
Check if a file is accessible either on the local host or on a remotely
located host.
srvObj: Instance of the server object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
fileId: File ID (string).
fileVersion: File Version (integer).
diskId: Disk ID (string).
Returns: Returns message indicating if the file is available
(string).
"""
T = TRACE()
logger.debug("Checking for access to file with ID: %s", fileId)
# Check if the file is located on this host, or if the request should be
# forwarded (if this server should act as proxy).
location, fileHost, ipAddress, filePortNo, mountPoint, filename, fileId,\
fileVersion, mimeType =\
ngamsFileUtils.locateArchiveFile(srvObj, fileId, fileVersion,
diskId)
if (location != NGAMS_HOST_LOCAL):
# Go and get it!
host, port = srvObj.get_remote_server_endpoint(fileHost)
pars = (('file_access', fileId), ('file_version', fileVersion),
('disk_id', diskId))
resp = ngamsHttpUtils.httpGet(host, port, 'STATUS', pars, timeout=60)
with contextlib.closing(resp):
return ngamsStatus.to_status(resp, fileHost, 'STATUS').getMessage()
else:
# First check if this system allows for Retrieve Requests.
if (not srvObj.getCfg().getAllowRetrieveReq()):
msg = genLog("NGAMS_INFO_SERVICE_UNAVAIL", ["File Retrieval"])
else:
fileHost = "%s:%d" % (getHostName(), filePortNo)
msg = genLog("NGAMS_INFO_FILE_AVAIL",
[fileId + "/Version: " + str(fileVersion), fileHost])
return msg
def handlePars(reqPropsObj, parDic):
"""
Parse/handle the HTTP parameters.
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
parDic: Dictionary with the parameters (dictionary).
Returns: Void.
"""
T = TRACE()
# Get parameters.
logger.debug("Get request parameters")
parDic[TARG_MIME_TYPE] = None
parDic[FILE_ID] = None
parDic[VERSIONING] = 1
parDic[COMPRESSION] = None
parDic[COMPRESSION_EXT] = None
if (reqPropsObj.hasHttpPar(TARG_MIME_TYPE)):
parDic[TARG_MIME_TYPE] = reqPropsObj.getHttpPar(TARG_MIME_TYPE)
# If the file_id is not given, we derive it from the name of the URI.
if (reqPropsObj.hasHttpPar(FILE_ID)):
parDic[FILE_ID] = reqPropsObj.getHttpPar(FILE_ID)
if (not parDic[FILE_ID]):
if (reqPropsObj.getFileUri().find("file_id=") > 0):
file_id = reqPropsObj.getFileUri().split("file_id=")[1]
parDic[FILE_ID] = file_id
logger.info("No file_id given, but found one in the URI: %s",
parDic[FILE_ID])
else:
parDic[FILE_ID] = os.path.basename(reqPropsObj.getFileUri())
logger.info("No file_id given, using basename of URI: %s",
parDic[FILE_ID])
if (reqPropsObj.hasHttpPar(VERSIONING)):
parDic[VERSIONING] = int(reqPropsObj.getHttpPar(VERSIONING))
# Set also the no_versioning parameter for backwards compatibility reasons
if (parDic[VERSIONING]):
reqPropsObj.addHttpPar("no_versioning", "0")
else:
reqPropsObj.addHttpPar("no_versioning", "1")
if (reqPropsObj.hasHttpPar(COMPRESSION)):
parDic[COMPRESSION] = reqPropsObj.getHttpPar(COMPRESSION)
if (reqPropsObj.hasHttpPar(COMPRESSION_EXT)):
parDic[COMPRESSION_EXT] = reqPropsObj.getHttpPar(COMPRESSION_EXT)
if ((parDic[COMPRESSION] and (parDic[COMPRESSION_EXT] == None)) or
(not parDic[COMPRESSION] and (parDic[COMPRESSION_EXT] != None))):
raise Exception, genLog("NGAMS_ER_DAPI", [
"Parameters compression and compression_ext"
"must be given together."
])
def _handleCmdCRetrieve(srvObj, reqPropsObj, httpRef):
"""
Carry out the action of a CRETRIEVE command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of
actions done during the request handling
(ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
# For data files, retrieval must be enabled otherwise the request is
# rejected.
if (not srvObj.getCfg().getAllowRetrieveReq()):
errMsg = genLog("NGAMS_ER_ILL_REQ", ["Retrieve"])
raise Exception(errMsg)
# We don't allow processing yet
if 'processing' in reqPropsObj:
raise Exception('CRETRIEVE command does not allow processing (yet)')
# At least container_id or container_name must be specified
containerName = containerId = None
if reqPropsObj.hasHttpPar("container_id") and reqPropsObj.getHttpPar(
"container_id").strip():
containerId = reqPropsObj.getHttpPar("container_id").strip()
if not containerId and reqPropsObj.hasHttpPar(
"container_name") and reqPropsObj.getHttpPar(
"container_name").strip():
containerName = reqPropsObj.getHttpPar("container_name").strip()
if not containerId and not containerName:
errMsg = genLog("NGAMS_ER_RETRIEVE_CMD")
raise Exception(errMsg)
# If container_name is specified, and maps to more than one container,
# an error is issued
if not containerId:
containerId = srvObj.getDb().getContainerIdForUniqueName(containerName)
logger.debug("Handling request for file with containerId: %s", containerId)
# Build the container hierarchy, get all file references and send back the results
container = srvObj.getDb().readHierarchy(containerId, True)
cinfo = cinfo_from_database(container, srvObj, reqPropsObj)
reader = ngamsMIMEMultipart.ContainerReader(cinfo)
# Send all the data back
httpRef.send_data(reader, NGAMS_CONT_MT)
def handleCmd(srvObj, reqPropsObj, httpRef):
"""
Handle REMDISK command. See also 'handleRemDisk()'.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
if (not srvObj.getCfg().getAllowRemoveReq()):
errMsg = genLog("NGAMS_ER_ILL_REQ", ["Remove"])
raise Exception(errMsg)
diskId = ""
execute = 0
if (reqPropsObj.hasHttpPar("disk_id")):
diskId = reqPropsObj.getHttpPar("disk_id")
if (diskId == ""):
errMsg = genLog("NGAMS_ER_CMD_SYNTAX",
[NGAMS_REMDISK_CMD, "Missing parameter: disk_id"])
raise Exception(errMsg)
if (not srvObj.getDb().diskInDb(diskId)):
errMsg = genLog("NGAMS_ER_UNKNOWN_DISK", [diskId])
raise Exception(errMsg)
if (reqPropsObj.hasHttpPar("execute")):
try:
execute = int(reqPropsObj.getHttpPar("execute"))
except:
errMsg = genLog("NGAMS_ER_REQ_HANDLING", ["Must provide proper " +\
"value for parameter: execute (0|1)"])
raise Exception(errMsg)
# Carry out the command.
status = remDisk(srvObj, reqPropsObj, diskId, execute)
# Send reply back to requestor (if not already done).
if (not status): return
xmlStat = status.genXmlDoc(0, 1, 1, 1, 0)
xmlStat = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, xmlStat)
if (status.getStatus() == NGAMS_SUCCESS):
httpStat = NGAMS_HTTP_SUCCESS
else:
httpStat = NGAMS_HTTP_BAD_REQ
httpRef.send_data(six.b(xmlStat), NGAMS_XML_MT, code=httpStat)
def extract_compression_params(reqPropsObj, plugin_pars):
plugin_pars[COMPRESSION] = None
plugin_pars[COMPRESSION_EXT] = None
if COMPRESSION in reqPropsObj:
plugin_pars[COMPRESSION] = reqPropsObj[COMPRESSION]
if COMPRESSION_EXT in reqPropsObj:
plugin_pars[COMPRESSION_EXT] = reqPropsObj[COMPRESSION_EXT]
if ((plugin_pars[COMPRESSION] and plugin_pars[COMPRESSION_EXT] is None) or
(not plugin_pars[COMPRESSION] and plugin_pars[COMPRESSION_EXT] is not None)):
raise Exception, genLog("NGAMS_ER_DAPI",
["Parameters compression and compression_ext"
"must be given together."])
def saveInStagingFile(ngamsCfgObj, reqPropsObj, httpRef, stagingFilename,
diskInfoObj):
"""
Save the data ready on the HTTP channel, into the given Staging
Area file.
ngamsCfgObj: NG/AMS Configuration (ngamsConfig).
reqPropsObj: NG/AMS Request Properties object (ngamsReqProps).
stagingFilename: Staging Area Filename as generated by
ngamsHighLevelLib.genStagingFilename() (string).
diskInfoObj: Disk info object. Only needed if mutual exclusion
is required for disk access (ngamsDiskInfo).
Returns: Void.
"""
T = TRACE()
try:
blockSize = ngamsCfgObj.getBlockSize()
return saveFromHttpToFile(ngamsCfgObj, reqPropsObj, httpRef,
stagingFilename, blockSize, 1, diskInfoObj)
except Exception as e:
errMsg = genLog("NGAMS_ER_PROB_STAGING_AREA",
[stagingFilename, str(e)])
logger.exception(errMsg)
raise
def specificTreatment(fo):
"""
Method contains the specific treatment of the file passed from NG/AMS.
fo: File object
Returns: (file_id, finalFileName, type);
The finalFileName is a string containing the name of the final
file without extension. type is the mime-type from the header.
"""
import rfc822, cgi, re
_EXT = '.msg'
filename = fo.name
# uidTempl = re.compile("[xX][0-9,a-f]{3,}/[xX][0-9,a-f]{1,}$")
# This matches the old UID of type X0123456789abcdef/X01234567
uidTempl = re.compile(
"^[uU][iI][dD]:/(/[xX][0-9,a-f,A-F]+){3}(#\w{1,}){0,}$")
# This matches the new UID structure uid://X1/X2/X3#kdgflf
try:
message = rfc822.Message(fo)
type, tparams = cgi.parse_header(message["Content-Type"])
except Exception, e:
err = "Parsing of mime message failed: " + str(e)
errMsg = genLog("NGAMS_ER_DAPI_BAD_FILE",
[os.path.basename(filename), _PLUGIN_ID, err])
raise Exception, errMsg
def checkFitsChecksum(reqPropsObj, stgFile=None):
"""
Carry out a check of the DATASUM and CHECKSUM for each HDU in the file.
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
stgFile: If specified this is taken rather than from the Request
Properties Object (ngamsReqProps).
Returns: Void.
"""
if (not stgFile): stgFile = reqPropsObj.getStagingFilename()
cmd = "chksumVerFitsChksum %s" % stgFile
stat, out = ngamsPlugInApi.execCmd(cmd)
if (out.find("Status: OK") == -1):
if (out.find("Status: NOT OK") != -1):
errMsg = genLog(
"NGAMS_ER_DAPI_BAD_FILE",
[stgFile, "checkFitsChecksum", "Illegal CHECKSUM/DATASUM"])
else:
errMsg = "Problem found while checking file: %s. " %\
out.replace("\n", " ")
# Backwards compatibility with old ESO FITS CHECKSUM scheme: If the
# file pass the previous scheme, we consider it as OK and do not
# return the failure.
cmd = "chksumGenChecksum %s" % stgFile
stat, out = ngamsPlugInApi.execCmd(cmd)
if (out.strip().find("0/0000000000000000") == -1):
raise Exception, errMsg
chksumUtil = "chksumGenChecksum"
else:
chksumUtil = "chksumVerFitsChksum"
logger.debug("File: %s checked with: %s. Result: OK", stgFile, chksumUtil)
def getFitsKeys(fitsFile, keyList):
"""
Get a FITS keyword from a FITS file. A dictionary is returned whereby
the keys in the keyword list are the dictionary keys and the value
the elements that these refer to.
fitsFile: Filename of FITS file (string).
keyList: Tuple of keys for which to extract values (tuple).
Returns: Dictionary with the values extracted of the format:
{<key 1>: [<val hdr 0>, <val hdr 1> ...], <key 2>: ...}
(dictionary).
"""
T = TRACE()
import astropy.io.fits as pyfits
keyDic = defaultdict(list)
try:
for key in keyList:
vals = pyfits.getval(fitsFile, key)
if isinstance(vals, basestring):
vals = [vals]
keyDic[key] = list(vals)
return keyDic
except Exception, e:
msg = ". Error: %s" % str(e)
errMsg = genLog("NGAMS_ER_RETRIEVE_KEYS",
[str(keyList), fitsFile + msg])
logger.exception(errMsg)
raise
def getDpIdInfo(filename):
"""
Generate the File ID (here DP ID) for the file.
filename: Name of FITS file (string).
Returns: Tuple containing the value of ARCFILE, the DP ID
of the file, and the JD date. The two latter deducted from
the ARCFILE keyword (tuple).
"""
try:
keyDic = getFitsKeys(filename, ["ARCFILE"])
arcFile = keyDic["ARCFILE"][0]
els = arcFile.split(".")
dpId = els[0] + "." + els[1] + "." + els[2]
date = els[1].split("T")[0]
# Make sure that the files are stored according to JD
# (one night is 12am -> 12am).
isoTime = '.'.join(els[1:])
ts1 = fromiso8601(isoTime)
ts2 = tomjd(ts1) - 0.5
dateDirName = toiso8601(frommjd(ts2), fmt=FMT_DATE_ONLY)
return [arcFile, dpId, dateDirName]
except:
err = "Did not find keyword ARCFILE in FITS file or ARCFILE illegal"
errMsg = genLog("NGAMS_ER_DAPI_BAD_FILE",
[os.path.basename(filename), "ngamsFitsPlugIn", err])
logger.exception(errMsg)
raise
def specificTreatment(fo):
"""
Method contains the specific treatment of the file passed from NG/AMS.
fo: File object
Returns: (file_id, finalFileName, type);
The finalFileName is a string containing the name of the final
file without extension. type is the mime-type from the header.
"""
import rfc822, cgi, re
_EXT = '.msg'
filename = fo.name
uidTempl = re.compile(
"^[uU][iI][dD]://[aAbBcCzZxX][0-9,a-z,A-Z]+(/[xX][0-9,a-z,A-Z]+){2}(#\w{1,}|/\w{0,}){0,}$"
)
try:
message = rfc822.Message(fo)
type, tparams = cgi.parse_header(message["Content-Type"])
except Exception, e:
err = "Parsing of mime message failed: " + str(e)
errMsg = genLog("NGAMS_ER_DAPI_BAD_FILE",
[os.path.basename(filename), _PLUGIN_ID, err])
raise Exception, errMsg
def checkTarball(filename):
"""
Check that the tarball is correct
filename: Name of tarball file (string).
Returns: Void.
"""
err = 0
tmpFilename = filename.replace(":", "_") + "_tmp"
try:
execCmd("ln -s " + filename + " " + tmpFilename)
stat, out, _ = execCmd("tar -tf " + tmpFilename)
rmFile(tmpFilename)
if (stat != 0):
errMsg = str(out).replace("\n", " --- ")
err = 1
except Exception as e:
rmFile(tmpFilename)
errMsg = str(e)
err = 1
if (err):
errMsg = "Error checking tarball: " + errMsg
errMsg = genLog("NGAMS_ER_DAPI_BAD_FILE",
[filename, "ngasTarBallPlugIn", errMsg])
raise Exception(errMsg)
def handleCmd(srvObj, reqPropsObj, httpRef):
"""
Handle UNSUBSCRIBE command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
# added by [email protected]
if (reqPropsObj.hasHttpPar("subscr_id")):
id = reqPropsObj.getHttpPar("subscr_id")
err, errStr = delSubscriber(srvObj, id)
###########
else:
if (reqPropsObj.hasHttpPar("url")):
url = reqPropsObj.getHttpPar("url")
else:
errMsg = genLog("NGAMS_ER_CMD_SYNTAX",
[NGAMS_SUBSCRIBE_CMD, "Missing parameter: url"])
raise Exception(errMsg)
err, errStr = delSubscriber(srvObj, ngamsLib.getSubscriberId(url))
if err:
httpRef.send_status('UNSUBSCRIBE command failed: ' + errStr,
status=NGAMS_FAILURE,
code=NGAMS_HTTP_SUCCESS)
else:
return "Successfully handled UNSUBSCRIBE command"
def ngamsRegisterGenericPlugIn(server_object, request_object, param_dict):
"""
Generic registration plug-in to handle registration of files
:param server_object: Reference to NG/AMS Server Object (ngamsServer)
:param request_object: NG/AMS request properties object (ngamsReqProps)
:param param_dict: Parameter dictionary
:return: Standard NG/AMS Data Archiving Plug-In Status as generated by ngamsPlugInApi.genDapiSuccessStat()
(ngamsDapiStatus)
"""
logger.info("Register generic plug-in registering file with URI: %s", request_object.getFileUri())
disk_info = request_object.getTargDiskInfo()
mime_type = request_object.getMimeType()
stage_file = request_object.getStagingFilename()
file_id = os.path.basename(request_object.getFileUri())
if mime_type == ngamsCore.NGAMS_UNKNOWN_MT:
error_message = ngamsCore.genLog("NGAMS_ER_UNKNOWN_MIME_TYPE1", [file_id])
raise Exception(error_message)
file_size = ngamsPlugInApi.getFileSize(stage_file)
compression = ""
uncompressed_size = file_size
# Get various information about the file being handled
file_version, relative_path, relative_filename, complete_filename, file_exists = \
ngamsPlugInApi.genFileInfoReg(server_object.getDb(), server_object.getCfg(), request_object, disk_info,
stage_file, file_id)
logger.info("Register generic plug-in finished processing file with URI %s: file_id=%s, file_version=%s, format=%s, file_size=%s",
request_object.getFileUri(), file_id, file_version, mime_type, file_size)
return ngamsPlugInApi.genRegPiSuccessStat(disk_info.getDiskId(), relative_filename, file_id, file_version,
mime_type, file_size, uncompressed_size, compression, relative_path,
disk_info.getSlotId(), file_exists, complete_filename)
def getDiskInfo(srvObj,
reqPropsObj = None):
"""
Invokes the plug-in listed in the configuration to extract the information
about the disks mounted. The information is returned in a dictionary which
contains the Slot IDs as keys, and elements which each is an object of type
ngamsPhysDiskInfo.
srvObj: Reference to NG/AMS Server class instance (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: Dictionary containing information about the
disks (dictionary).
"""
plugIn = srvObj.getCfg().getOnlinePlugIn()
logger.info("Invoking System Online Plug-In: %s(srvObj)", plugIn)
plugInMethod = loadPlugInEntryPoint(plugIn)
diskInfoDic = plugInMethod(srvObj, reqPropsObj)
if not diskInfoDic:
if (not ngamsLib.trueArchiveProxySrv(srvObj.getCfg())):
errMsg = genLog("NGAMS_NOTICE_NO_DISKS_AVAIL",
[srvObj.getHostId(),
srvObj.getHostId()])
logger.warning(errMsg)
logger.debug("Disk Dictionary: %s", str(diskInfoDic))
return diskInfoDic
def ngams_generic(ngams_server, request_properties):
"""
Data Archiving Plug-In to handle archiving of SDM multipart related message files containing ALMA UIDs in the
Content-Location mime parameter or any other kind of file
:param ngams_server: Reference to NG/AMS Server Object (ngamsServer)
:param request_properties: NG/AMS request properties object (ngamsReqProps)
:return: Standard NG/AMS Data Archiving Plug-In Status as generated by: ngamsPlugInApi.genDapiSuccessStat()
(ngamsDapiStatus)
"""
logger.info("Mirroring plug-in handling data for file: %s", os.path.basename(request_properties.getFileUri()))
# Create staging file
disk_info = request_properties.getTargDiskInfo()
staging_filename = request_properties.getStagingFilename()
# request_properties format: /MIRRARCHIVE?mime_type=application/x-tar&filename=...
file_format = request_properties.getMimeType()
if not file_format:
raise Exception("mime_type parameter not specified in MIRRARCHIVE request")
# Example of file URI format:
# http://ngas01.org:7777/RETRIEVE?disk_id=59622720f79296473f6106c15e5c2240&host_id=ngas01:7777&quick_location=1&file_version=1&file_id=backup.2011-02-02T22:01:59.tar
file_id = request_properties.fileinfo["fileId"]
file_version = request_properties.fileinfo["fileVersion"]
# Specific treatment depending on the mime-type
if file_format.find("multipart") >= 0 or file_format.find("multialma") >= 0:
logger.debug("Mirroring plug-in applying specific treatment for multipart/multialma mime file")
file_id, final_filename, file_format = specific_treatment(staging_filename)
else:
final_filename = file_id
logger.debug("Mirroring plug-in processing request for file with URI %s, file_format=%s, file_id=%s, "
"file_version=%s, final_filename=%s", request_properties.getFileUri(), file_format, file_id,
file_version, final_filename)
try:
# Compression parameters
uncompressed_size = ngamsPlugInApi.getFileSize(staging_filename)
compression = ""
today = ngamsCore.toiso8601(fmt=ngamsCore.FMT_DATE_ONLY)
relative_path, relative_filename, complete_filename, file_exists = \
generate_file_info(ngams_server.getCfg(), disk_info, staging_filename, file_version, final_filename, [today])
# Make sure the format is defined
if not file_format:
file_format = ngamsPlugInApi.determineMimeType(ngams_server.getCfg(), staging_filename)
file_size = ngamsPlugInApi.getFileSize(staging_filename)
return ngamsPlugInApi.genDapiSuccessStat(disk_info.getDiskId(), relative_filename, file_id, file_version,
file_format, file_size, uncompressed_size, compression, relative_path,
disk_info.getSlotId(), file_exists, complete_filename)
except Exception as e:
raise Exception(genLog("NGAMS_ER_DAPI_BAD_FILE",
[staging_filename, PLUGIN_ID, "Problem processing file in staging area: " + str(e)]))
def handleCmd(srvObj, reqPropsObj, httpRef):
"""
Handle REMFILE command. See also 'handleRemFile()'.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
T = TRACE()
if (not srvObj.getCfg().getAllowRemoveReq()):
errMsg = genLog("NGAMS_ER_ILL_REQ", ["Remove"])
raise Exception(errMsg)
diskId = ""
fileId = ""
fileVersion = -1
execute = 0
if (reqPropsObj.hasHttpPar("disk_id")):
diskId = reqPropsObj.getHttpPar("disk_id")
if (reqPropsObj.hasHttpPar("file_id")):
fileId = reqPropsObj.getHttpPar("file_id")
if (reqPropsObj.hasHttpPar("file_version")):
fileVersion = int(reqPropsObj.getHttpPar("file_version"))
if (reqPropsObj.hasHttpPar("execute")):
try:
execute = int(reqPropsObj.getHttpPar("execute"))
except:
errMsg = genLog("NGAMS_ER_REQ_HANDLING", ["Must provide proper " +\
"value for parameter: execute (0|1)"])
raise Exception(errMsg)
# Carry out the command.
status = remFile(srvObj, reqPropsObj, diskId, fileId, fileVersion, execute)
# Send reply back to requestor.
xmlStat = status.genXmlDoc(0, 1, 1, 1, 0)
xmlStat = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, xmlStat)
httpRef.send_data(xmlStat, NGAMS_XML_MT)
def ngamsGenDapi(srvObj, reqPropsObj):
"""
Generic Data Archiving Plug-In to handle archiving of any file.
srvObj: Reference to NG/AMS Server Object (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: Standard NG/AMS Data Archiving Plug-In Status
as generated by: ngamsPlugInApi.genDapiSuccessStat()
(ngamsDapiStatus).
"""
# For now the exception handling is pretty basic:
# If something goes wrong during the handling it is tried to
# move the temporary file to the Bad Files Area of the disk.
logger.debug("Plug-In handling data for file: %s",
os.path.basename(reqPropsObj.getFileUri()))
try:
parDic = {}
handlePars(reqPropsObj, parDic)
diskInfo = reqPropsObj.getTargDiskInfo()
stgFile = reqPropsObj.getStagingFilename()
ext = os.path.splitext(stgFile)[1][1:]
# Generate file information.
logger.debug("Generate file information")
dateDir = toiso8601(fmt=FMT_DATE_ONLY)
fileVersion, relPath, relFilename,\
complFilename, fileExists =\
ngamsPlugInApi.genFileInfo(srvObj.getDb(),
srvObj.getCfg(),
reqPropsObj, diskInfo,
reqPropsObj.\
getStagingFilename(),
parDic[FILE_ID],
parDic[FILE_ID], [dateDir])
complFilename, relFilename = checkForDblExt(complFilename, relFilename)
# Compress the file if requested.
uncomprSize, archFileSize, format, compression, comprExt =\
compressFile(srvObj, reqPropsObj, parDic)
if (comprExt != ""):
complFilename += ".%s" % comprExt
relFilename += ".%s" % comprExt
logger.debug(
"DAPI finished processing file - returning to host application")
return ngamsPlugInApi.genDapiSuccessStat(
diskInfo.getDiskId(), relFilename, parDic[FILE_ID], fileVersion,
format, archFileSize, uncomprSize, compression, relPath,
diskInfo.getSlotId(), fileExists, complFilename)
except Exception as e:
msg = "Error occurred in DAPI: %s" % str(e)
logger.error(msg)
raise Exception(genLog("NGAMS_ER_DAPI_RM", [msg]))
def specific_treatment(file_path):
"""
Method contains the specific treatment of the file passed from NG/AMS
:param file_path: File path
:return: (file_id, final_filename, file_type); The final_filename is a string containing the name of the final
file without extension. file_type is the mime-type from the header.
"""
filename = os.path.basename(file_path)
mime_message = parse_multipart_primary_header(file_path)
if mime_message is None:
raise Exception(genLog("NGAMS_ER_DAPI_BAD_FILE",
[filename, PLUGIN_ID, "Failed to parse MIME message"]))
file_type = mime_message.get_content_type()
alma_uid = mime_message["alma-uid"]
if alma_uid is None:
alma_uid = mime_message["Content-Location"]
if alma_uid is None:
raise Exception(genLog("NGAMS_ER_DAPI_BAD_FILE",
[filename, PLUGIN_ID,
"Mandatory 'alma-uid' and/or 'Content-Location' parameter not found in MIME header!"]))
if UID_EXPRESSION.match(alma_uid) is None:
raise Exception(genLog("NGAMS_ER_DAPI_BAD_FILE",
[filename, PLUGIN_ID, "Invalid alma-uid found in Content-Location: " + alma_uid]))
# Now, build final filename. We do that by looking for the UID in the message mime-header.
# The final filename is built as follows: <ALMA-UID>.<EXT> where ALMA-UID has the slash character in the UID
# replaced by colons.
try:
# Get rid of the 'uid://' and of anything following a '#' sign
alma_uid = alma_uid.split("//", 2)[1].split("#")[0]
alma_uid = alma_uid.replace("x", "X")
file_id = alma_uid
final_filename = alma_uid.replace("/", ":")
except Exception as e:
raise Exception(genLog("NGAMS_ER_DAPI_BAD_FILE",
[filename, PLUGIN_ID, "Problem constructing final file name: " + str(e)]))
return file_id, final_filename, file_type
def ngamsSdmMultipart(ngams_server, request_properties):
"""
Data Archiving Plug-In to handle archiving of SDM multipart related message files containing ALMA UIDs in the
Content-Location mime parameter
:param ngams_server: Reference to NG/AMS Server Object (ngamsServer)
:param request_properties: NG/AMS request properties object (ngamsReqProps)
:return: Standard NG/AMS Data Archiving Plug-In Status as generated by: ngamsPlugInApi.genDapiSuccessStat()
(ngamsDapiStatus)
"""
# For now the exception handling is pretty basic:
# If something goes wrong during the handling it is tried to move the temporary file to the bad-files directory
logger.info("SDM multipart plug-in handling data for file: %s", os.path.basename(request_properties.getFileUri()))
disk_info = request_properties.getTargDiskInfo()
staging_filename = request_properties.getStagingFilename()
file_id, final_filename, file_format = specific_treatment(staging_filename)
if request_properties.hasHttpPar("file_id"):
file_id = request_properties.getHttpPar("file_id")
#if request_properties.hasHttpPar("file_version"):
# file_version = request_properties.getHttpPar("file_version")
logger.debug("SDM multipart plug-in processing request for file with URI %s, file_format=%s, file_id=%s, "
"final_filename=%s", request_properties.getFileUri(), file_format, file_id, final_filename)
try:
# Compression parameters
uncompressed_size = ngamsPlugInApi.getFileSize(staging_filename)
compression = ""
# Remember to update the temporary file name in the request properties object
request_properties.setStagingFilename(staging_filename)
today = ngamsCore.toiso8601(fmt=ngamsCore.FMT_DATE_ONLY)
file_version, relative_path, relative_filename, complete_filename, file_exists = \
ngamsPlugInApi.genFileInfo(ngams_server.getDb(), ngams_server.getCfg(), request_properties, disk_info,
staging_filename, file_id, final_filename, [today])
# Make sure the format is defined
if not file_format:
file_format = ngamsPlugInApi.determineMimeType(ngams_server.getCfg(), staging_filename)
file_size = ngamsPlugInApi.getFileSize(staging_filename)
return ngamsPlugInApi.genDapiSuccessStat(disk_info.getDiskId(), relative_filename, file_id, file_version,
file_format, file_size, uncompressed_size, compression, relative_path,
disk_info.getSlotId(), file_exists, complete_filename)
except Exception as e:
raise Exception(genLog("NGAMS_ER_DAPI_BAD_FILE",
[staging_filename, PLUGIN_ID, "Problem processing file in staging area: " + str(e)]))
def __handleCmd(srvObj, reqPropsObj):
"""
Handle the Mirroring Archive (MIRRARCHIVE) Command.
srvObj: Reference to NG/AMS server class object (ngamsServer).setState
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
Returns: Void.
"""
TRACE()
# Is this NG/AMS permitted to handle Archive Requests?
logger.debug(
"Checking if this NG/AMS permitted to handle Archive Requests?")
if (not srvObj.getCfg().getAllowArchiveReq()):
errMsg = genLog("NGAMS_ER_ILL_REQ", ["Archive"])
raise Exception, errMsg
# Generate staging filename.
stgFilename = reqPropsObj.getStagingFilename()
logger.info("staging filename is: %s", stgFilename)
startByte = 0
if (os.path.exists(stgFilename) == 0):
logger.debug('this is a new staging file')
else:
startByte = os.path.getsize(stgFilename)
logger.debug(
'staging file already exists, requesting resumption of download from byte %d',
startByte)
# Set reference in request handle object to the read socket.
try:
# Retrieve file_id and file_version from request proposal
file_id = reqPropsObj.fileinfo['fileId']
file_version = reqPropsObj.fileinfo['fileVersion']
logger.debug("Got file_id=%s and file_version=%s", file_id,
file_version)
# Retrieve file contents (from URL, archive pull, or by storing the body
# of the HTTP request, archive push).
logger.info("Saving in staging file: %s", stgFilename)
stagingInfo = saveInStagingFile(srvObj, srvObj.getCfg(), reqPropsObj,
stgFilename, startByte)
reqPropsObj.incIoTime(stagingInfo[0])
checksumPlugIn = "ngamsGenCrc32"
checksum = stagingInfo[1]
except ngamsFailedDownloadException.FailedDownloadException, e:
raise
def handleCmd(srvObj, reqPropsObj, httpRef):
"""
Handles the CAPPEND command
@param srvObj: ngamsServer.ngamsServer
@param reqPropsObj: ngamsLib.ngamsReqProps
@param httpRef: ngamsLib.ngamsHttpRequestHandler
"""
# Check that we have been given either a containerId or a containerName
containerId = containerName = None
if reqPropsObj.hasHttpPar("container_id") and reqPropsObj.getHttpPar("container_id").strip():
containerId = reqPropsObj.getHttpPar("container_id").strip()
elif reqPropsObj.hasHttpPar("container_name") and reqPropsObj.getHttpPar("container_name").strip():
containerName = reqPropsObj.getHttpPar("container_name").strip()
if not containerId and not containerName:
errMsg = genLog("NGAMS_ER_RETRIEVE_CMD")
raise Exception(errMsg)
# Check if we have been asked to force the operation
force = False
if reqPropsObj.hasHttpPar('force') and reqPropsObj.getHttpPar('force') == '1':
force = True
# Check if we have been asked to "close" the container
closeContainer = False
if reqPropsObj.hasHttpPar('close_container') and reqPropsObj.getHttpPar('close_container') == '1':
closeContainer = True
# If container_name is specified, and maps to more than one container,
# (or to none) an error is issued
containerIdKnownToExist = False
if not containerId:
containerId = srvObj.getDb().getContainerIdForUniqueName(containerName)
containerIdKnownToExist = True
# If necessary, check that the container exists
if not containerIdKnownToExist:
if not srvObj.getDb().containerExists(containerId):
msg = "No container with containerId '" + containerId + "' found, cannot append files to it"
raise Exception(msg)
# If a single fileId has been given via URL parameters
# and the request is a GET we update that single file
# Otherwise, we assume a list of files is given in the
# body of he request
if reqPropsObj.getHttpMethod() == NGAMS_HTTP_GET:
_handleSingleFile(srvObj, containerId, reqPropsObj, force, closeContainer)
else:
_handleFileList(srvObj, containerId, reqPropsObj, httpRef, force, closeContainer)
def handleCmd(srvObj,
reqPropsObj,
httpRef):
"""
Handle REARCHIVE command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler object
(ngamsHttpRequestHandler).
Returns: Void.
"""
archive_start = time.time()
# Execute the init procedure for the ARCHIVE Command.
mimeType = ngamsArchiveUtils.archiveInitHandling(srvObj, reqPropsObj, httpRef)
# If mime-type is None, the request has been handled, i.e., it might have
# been a probe request or the server acting as proxy.
if (not mimeType): return
logger.debug("Archiving file: %s with mime-type: %s",
reqPropsObj.getSafeFileUri(), mimeType)
fileInfoObj, trgDiskInfoObj = receiveData(srvObj, reqPropsObj, httpRef)
processRequest(srvObj, reqPropsObj, httpRef, fileInfoObj, trgDiskInfoObj)
# If running as a cache archive, update the Cache New Files DBM
# with the information about the new file.
if (srvObj.getCachingActive()):
fileVer = fileInfoObj.getFileVersion()
ngamsCacheControlThread.addEntryNewFilesDbm(srvObj,
trgDiskInfoObj.getDiskId(),
fileInfoObj.getFileId(),
fileVer,
fileInfoObj.getFilename())
# Create log/syslog entry for the successfulyl handled request.
msg = genLog("NGAMS_INFO_FILE_ARCHIVED", [reqPropsObj.getSafeFileUri()])
msg = msg + ". Time: %.6fs" % (time.time() - archive_start)
logger.info(msg, extra={'to_syslog': True})
srvObj.setSubState(NGAMS_IDLE_SUBSTATE)
httpRef.send_ingest_status(msg, trgDiskInfoObj)
def notifyRegistrationService(srvObj, svrStatus='online'):
"""
to notify the ngas registration service that I am online now
svrStatus = online|offline
"""
parDicOnline = ngamsPlugInApi.\
parseRawPlugInPars(srvObj.getCfg().getOnlinePlugInPars())
if (parDicOnline.has_key("regsvr_host")):
if (svrStatus == "online"):
errTag = "NGAMS_ER_ONLINE_PLUGIN"
else:
errTag = "NGAMS_ER_OFFLINE_PLUGIN"
regsvr_host = parDicOnline["regsvr_host"]
regsvr_port = parDicOnline["regsvr_port"]
regsvr_path = parDicOnline["regsvr_path"]
host_name = getHostName()
host_port = srvObj.getCfg().getPortNo()
body = urllib.urlencode({
'ngas_host': host_name,
'ngas_port': host_port,
'status': svrStatus
})
hdrs = {"Accept": "text/plain"}
resp = ngamsHttpUtils.httpPost(
regsvr_host,
regsvr_port,
regsvr_path,
body,
mimeType='application/x-www-form-urlencoded',
hdrs=hdrs,
timeout=10)
with contextlib.closing(resp):
if resp.status != 200:
errMsg = "Problem notifying registration service! Error " + resp.reason
errMsg = genLog(errTag, [errMsg])
logger.error(errMsg)
#raise Exception, errMsg
else:
logger.debug("Successfully notified registration service: %s",
svrStatus)
logger.info(resp.read())
return
def handleCmd(srvObj,
reqPropsObj,
httpRef):
"""
Handle CLONE command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
T = TRACE()
# Is this NG/AMS permitted to handle Archive Requests?
if (not srvObj.getCfg().getAllowArchiveReq()):
errMsg = genLog("NGAMS_ER_ILL_REQ", ["Clone"])
raise Exception(errMsg)
# Check if State/Sub-State correct for perfoming the cloning.
srvObj.checkSetState("Command CLONE", [NGAMS_ONLINE_STATE],
[NGAMS_IDLE_SUBSTATE, NGAMS_BUSY_SUBSTATE])
# Get the parameters from the query.
if (reqPropsObj.hasHttpPar("file_id")):
fileId = reqPropsObj.getHttpPar("file_id")
else:
fileId =""
if (reqPropsObj.hasHttpPar("disk_id")):
diskId = reqPropsObj.getHttpPar("disk_id")
else:
diskId = ""
if (reqPropsObj.hasHttpPar("file_version")):
fileVersion = int(reqPropsObj.getHttpPar("file_version"))
else:
fileVersion = -1
if (reqPropsObj.hasHttpPar("target_disk_id")):
targetDiskId = reqPropsObj.getHttpPar("target_disk_id")
else:
targetDiskId = ""
# Carry out the cloning.
clone(srvObj, diskId, fileId, fileVersion,targetDiskId,reqPropsObj,httpRef)
def _handleCmdCRetrieve(srvObj,
reqPropsObj,
httpRef):
"""
Carry out the action of a CRETRIEVE command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of
actions done during the request handling
(ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
# For data files, retrieval must be enabled otherwise the request is
# rejected.
if (not srvObj.getCfg().getAllowRetrieveReq()):
errMsg = genLog("NGAMS_ER_ILL_REQ", ["Retrieve"])
raise Exception(errMsg)
# We don't allow processing yet
if 'processing' in reqPropsObj:
raise Exception('CRETRIEVE command does not allow processing (yet)')
# Users can request a tarball instead of the default MIME multipart message
return_tar = 'format' in reqPropsObj and reqPropsObj['format'] == 'application/x-tar'
container_id = containers.get_container_id(reqPropsObj, srvObj.db)
logger.debug("Handling request for file with containerId: %s", container_id)
# Build the container hierarchy and get all file references
container = srvObj.getDb().readHierarchy(container_id, True)
cinfo = cinfo_from_database(container, srvObj, reqPropsObj)
# Send all the data back, either as a multipart message or as a tarball
if return_tar:
httpRef.send_file_headers(cinfo.name, 'application/x-tar', tarsize_cinfo(cinfo))
send_toplevel_cinfo(cinfo, httpRef)
else:
reader = ngamsMIMEMultipart.ContainerReader(cinfo)
httpRef.send_data(reader, NGAMS_CONT_MT)