def handleCmd(srvObj, reqPropsObj, httpRef):
"""
Handle Command MIRRTABLE to populate bookkeeping table in target cluster
INPUT:
srvObj: ngamsServer, Reference to NG/AMS server class object
reqPropsObj: ngamsReqProps, Request Property object to keep track
of actions done during the request handling
httpRef: ngamsHttpRequestHandler, Reference to the HTTP request
handler object
RETURNS: Void.
"""
TRACE()
# Get command parameters.
mirror_cluster = 0
n_threads = 2
rx_timeout = None
if (reqPropsObj.hasHttpPar("mirror_cluster")):
mirror_cluster = int(reqPropsObj.getHttpPar("mirror_cluster"))
if (reqPropsObj.hasHttpPar("n_threads")):
n_threads = int(reqPropsObj.getHttpPar("n_threads"))
if (reqPropsObj.hasHttpPar("rx_timeout")):
rx_timeout = int(reqPropsObj.getHttpPar("rx_timeout"))
current_iteration = int(reqPropsObj.getHttpPar("iteration"))
# Distributed cluster mirroring
if (mirror_cluster):
# Get cluster name
local_cluster_name = get_cluster_name(srvObj)
# Get active target nodes
active_target_nodes = get_active_target_nodes(local_cluster_name,
current_iteration,
srvObj)
# Start mirroring
distributed_mirroring(active_target_nodes, n_threads, rx_timeout,
current_iteration)
else:
# Get full qualified name of this server
local_server_full_qualified_name = get_full_qualified_name(srvObj)
# Format full qualified name as a list
active_target_nodes = [local_server_full_qualified_name]
# Get active source nodes
active_source_nodes = get_active_source_nodes(
srvObj,
current_iteration,
full_qualified_name=local_server_full_qualified_name)
# Start mirroring process driven by this host
logger.info(
"Performing mirroring tasks from (%s) to (%s) using %s threads per source node and target node",
str(active_source_nodes), str(active_target_nodes), str(n_threads))
try:
# Set mirroring running flag to avoid data check thread and janitor thread
srvObj.mirroring_running = True
multithreading_mirroring(active_source_nodes, n_threads,
rx_timeout, current_iteration, srvObj)
finally:
# Set mirroring running flag to trigger data check thread and janitor thread
srvObj.mirroring_running = False
# Return Void
return
def register(srvObj,
path,
mimeType = "",
reqPropsObj = None,
httpRef = None):
"""
Function to generate a list of candidate files to register, and to
launch a thread that actually carries out the registration.
When the possible candidate files haven been selected, a reply is sent
back to the requestor if the 'httpRef' object is given.
srvObj: Instance of NG/AMS Server object (ngamsServer).
path: The path name, which is used as starting point for
the searching for files (string).
mimeType: Comma separated list of mime-types, which should be
considered for registration (string).
reqPropsObj: If an NG/AMS Request Properties Object is given, the
Request Status will be updated as the request is carried
out (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
T = TRACE()
# Check if the given path or file exists.
if (not os.path.exists(path)):
errMsg = genLog("NGAMS_ER_FILE_REG_FAILED",
[path, "Non-existing file or path."])
raise Exception, errMsg
# Check if the given path/file is on one of the NGAS Disks.
foundPath = 0
for slotId in srvObj.getDiskDic().keys():
do = srvObj.getDiskDic()[slotId]
if (path.find(do.getMountPoint()) == 0):
foundPath = 1
break
if (not foundPath):
errMsg = genLog("NGAMS_ER_FILE_REG_FAILED",
[path, "File or path specified is not located on an "
"NGAS Disk."])
raise Exception, errMsg
# Create file pattern for temporary files.
tmpFilePat=ngamsHighLevelLib.genTmpFilename(srvObj.getCfg(),"REGISTER_CMD")
# Generate dictionary with mime-types to accept.
mimeTypeDic = {}
if (mimeType != ""):
for mt in mimeType.split(","):
try:
mimeTypeDic[mt] = srvObj.getMimeTypeDic()[mt]
except:
errMsg = genLog("NGAMS_ER_REQ_HANDLING",
["Mime-type specified: " + mt + " " +\
"in connection with REGISTER command " +\
"does not have a DAPI defined"])
raise Exception, errMsg
else:
mimeTypeDic = srvObj.getMimeTypeDic()
# From the Disk Dictionary, generate a dictionary with mappings from
# Disk ID + from Mount Point to the disk information for the disk.
diskInfoDic = {}
mtPt2DiskInfo = {}
for slotId in srvObj.getDiskDic().keys():
if (not srvObj.getCfg().getSlotIdDefined(slotId)): continue
diskId = srvObj.getDiskDic()[slotId].getDiskId()
mtPt = srvObj.getDiskDic()[slotId].getMountPoint()
tmpDiskInfo = ngamsDiskInfo.ngamsDiskInfo().\
read(srvObj.getDb(), diskId)
diskInfoDic[diskId] = tmpDiskInfo
mtPt2DiskInfo[mtPt] = tmpDiskInfo
# Generate a list with all files found under the specified path, which
# are candidates for being registered.
fileListDbmName = tmpFilePat + "_FILE_LIST"
rmFile(fileListDbmName + "*")
fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm = 1)
mimeTypeMappings = srvObj.getCfg().getMimeTypeMappings()
tmpGlobFile = tmpFilePat + "_FILE_GLOB.glob"
fileCount = 0
searchPath = os.path.normpath(path)
foundVolume = False
for mtPt in mtPt2DiskInfo.keys():
mtPt2 = mtPt
if (mtPt[-1] != "/"): mtPt2 += "/"
if (searchPath.find(mtPt2) == 0):
foundVolume = True
if os.path.isdir(searchPath):
for root, dirs, files in os.walk(searchPath):
for nextFile in files:
if nextFile not in \
[NGAMS_DISK_INFO, \
NGAMS_VOLUME_ID_FILE, \
NGAMS_VOLUME_INFO_FILE]:
mimeType = ngamsLib.detMimeType(mimeTypeMappings,
nextFile, 0)
tmpFileInfo = [os.path.join(root,nextFile),
mtPt2DiskInfo[mtPt].getDiskId(),
mimeType]
fileListDbm.add(os.path.join(root,nextFile), tmpFileInfo)
elif os.path.isfile(searchPath): # the path is actually pointing to a file
nextFile = os.path.basename(searchPath)
root = os.path.dirname(searchPath)
if nextFile not in \
[NGAMS_DISK_INFO, \
NGAMS_VOLUME_ID_FILE, \
NGAMS_VOLUME_INFO_FILE]:
mimeType = ngamsLib.detMimeType(mimeTypeMappings,
nextFile, 0)
tmpFileInfo = [os.path.join(root,nextFile),
mtPt2DiskInfo[mtPt].getDiskId(),
mimeType]
fileListDbm.add(os.path.join(root,nextFile), tmpFileInfo)
if foundVolume: break
# pattern = ""
# # TODO: Portatibility issue. The usage of UNIX commands should be
# # avoided if possible.
# while (1):
# # Use a shell commands here to avoid building up maybe huge lists in
# # memory/in the Python interpreter. A better way could maybe be found
# # avoiding to invoke a shell command.
# rmFile(tmpGlobFile)
# searchPath = os.path.normpath(path + pattern)
# tmpCmd = "find " + searchPath + " -maxdepth 1 > " + tmpGlobFile
# stat, out = commands.getstatusoutput(tmpCmd)
# if (stat != 0): break
# fo = open(tmpGlobFile)
# while (1):
# nextFile = fo.readline().strip()
# if (nextFile == ""): break
# if (nextFile.find("/" + NGAMS_DISK_INFO) != -1): continue
# if (nextFile.find("/" + NGAMS_VOLUME_ID_FILE) != -1): continue
# if (nextFile.find("/" + NGAMS_VOLUME_INFO_FILE) != -1): continue
# if (os.path.isfile(nextFile)):
# nextFile = os.path.normpath(nextFile)
#
# # Find the host disk. Only files located on mounted NGAS
# # disks are considered.
# for mtPt in mtPt2DiskInfo.keys():
# mtPt2 = mtPt
# if (mtPt[-1] != "/"): mtPt2 += "/"
# if (nextFile.find(mtPt2) == 0):
# info(4,"Found candidate file for registering: " +\
# nextFile)
# # Take only files with a Registration Plug-In defined.
# mimeType = ngamsLib.detMimeType(mimeTypeMappings,
# nextFile, 0)
# regPi = srvObj.getCfg().getRegPiFromMimeType(mimeType)
# if (regPi != None):
# tmpFileInfo = [nextFile,
# mtPt2DiskInfo[mtPt].getDiskId(),
# mimeType]
# fileListDbm.add(nextFile, tmpFileInfo)
# break
# fo.close()
# pattern += "/*"
# rmFile(tmpGlobFile)
fileListDbm.sync()
del fileListDbm
# Send intermediate reply if the HTTP Reference object is given.
async = 'async' in reqPropsObj and int(reqPropsObj['async'])
if httpRef and async:
logger.debug("REGISTER command accepted - generating immediate " +\
"confimation reply to REGISTER command")
# Update the request status in the Request Properties Object.
reqPropsObj.\
setExpectedCount(fileCount).\
setActualCount(0).setCompletionPercent(0)
srvObj.updateRequestDb(reqPropsObj)
status = srvObj.genStatus(NGAMS_SUCCESS,
"Accepted REGISTER command for execution").\
setReqStatFromReqPropsObj(reqPropsObj).\
setActualCount(0)
# Launch the register thread or run the command in foreground if wait=1
if async:
args = (srvObj, fileListDbmName, tmpFilePat, diskInfoDic,
reqPropsObj, None)
thrName = NGAMS_REGISTER_THR + threading.current_thread().getName()
regThread = threading.Thread(None, _registerThread, thrName, args)
regThread.setDaemon(0)
regThread.start()
else:
# Carry out the REGISTER Command (directly in this thread) and send
# reply when this is done.
_registerExec(srvObj, fileListDbmName, tmpFilePat, diskInfoDic,
reqPropsObj)
msg = "Successfully handled command REGISTER"
logger.debug(msg)
status = srvObj.genStatus(NGAMS_SUCCESS, msg).\
setReqStatFromReqPropsObj(reqPropsObj).setActualCount(0)
# Send reply if possible.
if (httpRef):
xmlStat = status.genXmlDoc(0, 0, 0, 1, 0)
xmlStat = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, xmlStat)
httpRef.send_data(xmlStat, NGAMS_XML_MT)
def _httpPostUrl(url,
mimeType,
contDisp="",
dataRef="",
dataSource="BUFFER",
dataTargFile="",
blockSize=65536,
suspTime=0.0,
timeOut=None,
authHdrVal="",
dataSize=-1,
session_uuid=""):
"""
Post the the data referenced on the given URL. This function is adapted from
ngamsLib.httpPostUrl, which does not support block-level suspension and cancelling for file transfer
The data send back from the remote server + the HTTP header information
is return in a list with the following contents:
[<HTTP status code>, <HTTP status msg>, <HTTP headers (list)>, <data>]
url: URL to where data is posted (string).
mimeType: Mime-type of message (string).
contDisp: Content-Disposition of the data (string).
dataRef: Data to post or name of file containing data to send
(string).
dataSource: Source where to pick up the data (string/BUFFER|FILE|FD).
dataTargFile: If a filename is specified with this parameter, the
data received is stored into a file of that name (string).
blockSize: Block size (in bytes) used when sending the data (integer).
suspTime: Time in seconds to suspend between each block (double).
timeOut: Timeout in seconds to wait for replies from the server
(double).
authHdrVal: Authorization HTTP header value as it should be sent in
the query (string).
dataSize: Size of data to send if read from a socket (integer).
Returns: List with information from reply from contacted
NG/AMS Server (reply, msg, hdrs, data) (list).
"""
T = TRACE()
# Separate the URL from the command.
idx = (url[7:].find("/") + 7)
tmpUrl = url[7:idx]
cmd = url[(idx + 1):]
http = httplib.HTTP(tmpUrl)
logger.debug("HTTP Header: %s: %s", NGAMS_HTTP_POST, cmd)
http.putrequest(NGAMS_HTTP_POST, cmd)
logger.debug("HTTP Header: Content-Type: %s", mimeType)
http.putheader("Content-Type", mimeType)
if (contDisp != ""):
logger.debug("HTTP Header: Content-Disposition: %s", contDisp)
http.putheader("Content-Disposition", contDisp)
if (authHdrVal):
if (authHdrVal[-1] == "\n"): authHdrVal = authHdrVal[:-1]
logger.debug("HTTP Header: Authorization: %s", authHdrVal)
http.putheader("Authorization", authHdrVal)
if (dataSource == "FILE"):
dataSize = getFileSize(dataRef)
elif (dataSource == "BUFFER"):
dataSize = len(dataRef)
if (dataSize != -1):
logger.debug("HTTP Header: Content-Length: %s", str(dataSize))
http.putheader("Content-Length", str(dataSize))
logger.debug("HTTP Header: Host: %s", getHostName())
http.putheader("Host", getHostName())
http.endheaders()
logger.debug("HTTP header sent")
http._conn.sock.settimeout(timeOut)
# Send the data.
logger.debug("Sending data ...")
if (dataSource == "FILE"):
fdIn = open(dataRef)
block = "-"
blockAccu = 0
http._conn.sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF,
87380) # to fit Fornax
while (block != ""):
"""
if (threadRunDic.has_key(session_uuid) and threadRunDic[session_uuid] == 0):
info(3, "Received cancel/suspend request, discard remaining blocks")
break
"""
block = fdIn.read(blockSize)
blockAccu += len(block)
http._conn.sock.sendall(block)
if (suspTime > 0.0): time.sleep(suspTime)
fdIn.close()
elif (dataSource == "FD"):
fdIn = dataRef
dataRead = 0
while (dataRead < dataSize):
if ((dataSize - dataRead) < blockSize):
rdSize = (dataSize - dataRead)
else:
rdSize = blockSize
block = fdIn.read(rdSize)
http._conn.sock.sendall(block)
dataRead += len(block)
if (suspTime > 0.0): time.sleep(suspTime)
else:
# dataSource == "BUFFER"
http.send(dataRef)
logger.debug("Data sent")
if (threadRunDic.has_key(session_uuid)
and threadRunDic[session_uuid] == 0):
logger.debug(
"Received cancel/suspend request, close HTTP connection and return None values"
)
if (http != None):
http.close()
del http
return [None, None, None, None]
# Receive + unpack reply.
logger.debug("Waiting for reply ...")
reply, msg, hdrs = http.getreply()
if (hdrs == None):
errMsg = "Illegal/no response to HTTP request encountered!"
raise Exception, errMsg
if (hdrs.has_key("content-length")):
dataSize = int(hdrs["content-length"])
else:
dataSize = 0
if (dataTargFile == ""):
data = http.getfile().read(dataSize)
else:
fd = None
try:
data = dataTargFile
fd = open(dataTargFile, "w")
fd.write(http.getfile().read(dataSize))
fd.close()
except Exception, e:
if (fd != None): fd.close()
raise e
def compressFile(srvObj,
reqPropsObj,
parDic):
"""
Compress the file if required.
srvObj: Reference to NG/AMS Server Object (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
parDic: Dictionary with parameters for the DAPI. This is generated
with ngamsPlugInApi.parseDapiPlugInPars() (Dictionary).
Returns: Tupe containing uncompressed filesize, archived filesize
and the format (mime-type) of the resulting data file and
the compression method (NONE if the file is not compressed),
finally, the extension added by the compression if any
(tuple).
"""
T = TRACE()
stFn = reqPropsObj.getStagingFilename()
# If a compression application is specified, apply this.
uncomprSize = ngamsPlugInApi.getFileSize(stFn)
comprExt = ""
if _compress_data(parDic):
logger.debug("Compressing file using: %s ...", parDic[COMPRESSION])
compCmd = "%s %s" % (parDic[COMPRESSION], stFn)
compress_start = time.time()
logger.debug("Compressing file with command: %s", compCmd)
with open(os.devnull, 'w') as f:
exitCode = subprocess.call([parDic[COMPRESSION], stFn], stdout = f, stderr = f)
# If the compression fails, assume that it is because the file is not
# compressible (although it could also be due to lack of disk space).
if (exitCode == 0):
if (parDic[COMPRESSION_EXT]):
stFn = stFn + "." + parDic[COMPRESSION_EXT]
comprExt = parDic[COMPRESSION_EXT]
# Remember to update Staging Filename in the Request Properties
# Object.
reqPropsObj.setStagingFilename(stFn)
# Handle mime-type
if (parDic[TARG_MIME_TYPE]):
format = parDic[TARG_MIME_TYPE]
else:
format = ngamsPlugInApi.determineMimeType(srvObj.getCfg(),
stFn)
compression = parDic[COMPRESSION]
logger.debug("File compressed. Time: %.3fs", time.time() - compress_start)
else:
# Carry on with the original file. We take the original mime-type
# as the target mime-type.
format = reqPropsObj.getMimeType()
compression = NO_COMPRESSION
else:
# Handle mime-type
if (parDic[TARG_MIME_TYPE]):
format = parDic[TARG_MIME_TYPE]
else:
format = reqPropsObj.getMimeType()
compression = NO_COMPRESSION
archFileSize = ngamsPlugInApi.getFileSize(reqPropsObj.getStagingFilename())
return uncomprSize, archFileSize, format, compression, comprExt
def _discardFile(srvObj,
diskId = None,
fileId = None,
fileVersion = None,
hostId = None,
path = None,
execute = 0):
"""
Discard a file from the system. If a Disk ID + File ID + File Version is
given, the file must be stored on the contacted host in order to be
executed.
srvObj: Reference to NG/AMS server class object (ngamsServer).
diskId: ID of disk hosting file or None (string|None).
fileId: ID of file or None (string|None).
fileVersion: File Version or None (integer|None).
hostId: ID of host where file is stored or None (string|None).
path: Path of file or None (string|None).
execute: If set to 1 the specified filel will be removed
(integer/0|1).
tmpFilePat: File pattern for temporary files (string).
Returns: Message indicating status of the execute (string).
"""
T = TRACE()
# Check the given paramereters.
if (diskId and fileId and fileVersion):
ngasHostId = "%s:%d" % (getHostName(), srvObj.getCfg().getPortNo())
files = srvObj.getDb().getFileInfoFromFileId(fileId, fileVersion, diskId, ignore=None, dbCursor=False)
if not files:
err = genLog("NGAMS_ER_DISCARD_NOT_FOUND",
["Disk ID: %s/File ID: %s/File Version: %s" %\
(str(diskId), str(fileId), str(fileVersion)),
ngasHostId])
return err
# Can only execute the DISCARD Command locally.
fileInfo = files[0]
hostId = fileInfo[-2]
if (hostId != srvObj.getHostId()):
raise Exception("DISCARD Command can only be executed locally!")
mtPt = fileInfo[-1]
filename = os.path.normpath(mtPt + "/" + fileInfo[1])
_delFile(srvObj, filename, hostId, execute)
if (execute):
srvObj.getDb().deleteFileInfo(srvObj.getHostId(), diskId, fileId, fileVersion)
msg = genLog("NGAMS_INFO_DISCARD_OK",
["Disk ID: %s/File ID: %s/File Version: %s" %\
(str(diskId), str(fileId), str(fileVersion)),
ngasHostId])
else:
msg = genLog("NGAMS_INFO_DISCARD_GRANTED",
["Disk ID: %s/File ID: %s/File Version: %s" %\
(str(diskId), str(fileId), str(fileVersion)),
ngasHostId])
elif (path):
msg = _delFile(srvObj, path, hostId, execute)
else:
msg = "Correct syntax is: disk_id=ID&file_id=ID&file_version=VER or "+\
"path=PATH[&host_id=ID]"
raise Exception(genLog("NGAMS_ER_CMD_SYNTAX",[NGAMS_DISCARD_CMD, msg]))
return msg
def handleCmd(srvObj, reqPropsObj, httpRef):
"""
Handle SUBSCRIBE Command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
T = TRACE()
"""
if (srvObj.getDataMoverOnlyActive() and len(srvObj.getSubscriberDic()) > 0):
srvObj.reply(reqPropsObj, httpRef, NGAMS_HTTP_SUCCESS, NGAMS_FAILURE,
"Data Mover NGAS server can have only one subscriber. Use command USUBSCRIBE to update an existing subscriber.")
return
"""
priority = 10
url = ""
startDate = time.time()
filterPi = ""
filterPiPars = ""
if (reqPropsObj.hasHttpPar("priority")):
priority = reqPropsObj.getHttpPar("priority")
if (reqPropsObj.hasHttpPar("url")):
url = reqPropsObj.getHttpPar("url")
ngamsSubscriber.validate_url(url)
else:
errMsg = genLog("NGAMS_ER_CMD_SYNTAX",
[NGAMS_SUBSCRIBE_CMD, "Missing parameter: url"])
raise Exception, errMsg
if (reqPropsObj.hasHttpPar("start_date")):
tmpStartDate = reqPropsObj.getHttpPar("start_date").strip()
if tmpStartDate:
startDate = fromiso8601(tmpStartDate, local=True)
if (reqPropsObj.hasHttpPar("filter_plug_in")):
filterPi = reqPropsObj.getHttpPar("filter_plug_in")
if (reqPropsObj.hasHttpPar("plug_in_pars")):
filterPiPars = reqPropsObj.getHttpPar("plug_in_pars")
if (reqPropsObj.hasHttpPar("subscr_id")):
id = reqPropsObj.getHttpPar("subscr_id")
else:
id = ngamsLib.getSubscriberId(url)
logger.info("Creating subscription for files >= %s", toiso8601(startDate))
subscrObj = ngamsSubscriber.ngamsSubscriber(srvObj.getHostId(),
srvObj.getCfg().getPortNo(),
priority,
url,
startDate,
filterPi,
filterPiPars,
subscrId=id)
# supports concurrent file transfer, added by [email protected]
if (reqPropsObj.hasHttpPar("concurrent_threads")):
concurthrds = reqPropsObj.getHttpPar("concurrent_threads")
subscrObj.setConcurrentThreads(concurthrds)
# If the Start Date given in before the Last Ingestion Date, we
# reset the Last Ingestion Date
subscrStat = srvObj.getDb().getSubscriberStatus([subscrObj.getId()],
subscrObj.getHostId(),
subscrObj.getPortNo())
if subscrStat:
lastIngDate = subscrStat[0][1]
if startDate < lastIngDate:
subscrObj.setLastFileIngDate(None)
else:
subscrObj.setLastFileIngDate(lastIngDate)
# Register the Subscriber.
addSubscriber(srvObj, subscrObj)
# Trigger the Data Susbcription Thread to make it check if there are
# files to deliver to the new Subscriber.
srvObj.addSubscriptionInfo([], [subscrObj]).triggerSubscriptionThread()
return "Handled SUBSCRIBE command"
def _handleFileListReply(srvObj,
reqPropsObj,
httpRef,
fileListId,
maxElements=None):
"""
Extracts file information from a previously dumped file information
in connection with a STATUS?file_list request.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler object
(ngamsHttpRequestHandler).
fileListId: File List ID allocated to this request (string).
maxElements: Maximum number of elements to extract and return to the
requestor (integer).
Returns: Void.
"""
T = TRACE()
# Get the name of the DBM in which the information is stored.
dbmBaseName = STATUS_FILE_LIST_DBM_TAG % fileListId
dbmPat = os.path.normpath("%s/*%s*" %\
(ngamsHighLevelLib.\
getNgasTmpDir(srvObj.getCfg()),
dbmBaseName))
dbmMatches = glob.glob(dbmPat)
if (len(dbmMatches) < 1):
msg = "Referenced File List ID: %s in connection with " +\
"STATUS/file_list request, is not (or no longer) known"
raise Exception(msg % fileListId)
elif (len(dbmMatches) > 1):
msg = "Inconsistencies encountered in locating result set for " +\
"STATUS/file_list for referenced File List ID: %s"
raise Exception(msg % fileListId)
fileInfoDbmName = dbmMatches[0]
# Generate the NG/AMS Status Document, with a File List in it.
# Compress it on the fly.
fileListXmlDoc = ngamsHighLevelLib.\
genTmpFilename(srvObj.getCfg(),
"STATUS_FILE_LIST_XML.xml")
try:
tmpFileListDbm = ngamsDbm.ngamsDbm(fileInfoDbmName,
cleanUpOnDestr=0,
writePerm=1)
rmFile(fileListXmlDoc)
fo = open(fileListXmlDoc, "w")
if (not maxElements):
remainingObjects = 0
else:
remainingObjects = (tmpFileListDbm.getCount() - maxElements)
if (remainingObjects < 0):
remainingObjects = 0
fo.write(_fileListXmlHdr % (fileListId, str(remainingObjects)))
# Loop over the file info objects and write them into the file.
# take max the number of elements specified.
tmpFileListDbm.initKeyPtr()
elCount = 0
keyRefList = []
while (True):
# Have the requested number of elements been extracted?
if (maxElements):
if (elCount >= maxElements):
break
# Get the next key (if there are more elements).
key, fileInfo = tmpFileListDbm.getNext()
if (not key):
break
try:
# Write the File Status XML Element in the file.
tmpFileInfoObj = ngamsFileInfo.ngamsFileInfo().\
unpackSqlResult(fileInfo)
fileInfoXml = tmpFileInfoObj.genXml(storeDiskId = 1).\
toprettyxml(" ", "\n")[:-1]
fo.write("\n" + fileInfoXml)
except Exception as e:
msg = "Error creating STATUS/File List XML Document. " +\
"Error: %s" % str(e)
logger.error(msg)
raise
keyRefList.append(key)
elCount += 1
# Finish up the XML document, close the file.
fo.write(_fileListXmlFooter)
fo.close()
# Assume this type of file can always be compressed.
fileListXmlDoc = compressFile(fileListXmlDoc)
except:
rmFile("%s*" % fileInfoDbmName)
rmFile("%s*" % fileListXmlDoc)
raise
# Send the XML document back to the requestor.
try:
httpRef.send_file(fileListXmlDoc, NGAMS_GZIP_XML_MT)
# Remove the reported entries.
for key in keyRefList:
tmpFileListDbm.rem(key)
tmpFileListDbm.sync()
del keyRefList
keyRefList = []
# If there are no more entries, delete the DBM.
dbmCount = tmpFileListDbm.getCount()
del tmpFileListDbm
if (dbmCount == 0):
rmFile("%s*" % fileInfoDbmName)
except Exception as e:
msg = "Error returning response to STATUS?file_list request. Error: %s"
msg = msg % str(e)
raise Exception(msg)
finally:
rmFile(fileListXmlDoc)
def archiveFromFile(srvObj,
filename,
noReplication=0,
mimeType=None,
reqPropsObj=None):
"""
Archive a file directly from a file as source.
srvObj: Reference to NG/AMS Server Object (ngamsServer).
filename: Name of file to archive (string).
noReplication: Flag to enable/disable replication (integer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
Returns: Execution result object of DAPI
"""
T = TRACE()
logger.debug("Archiving file: %s", filename)
logger.debug("Mimetype used is %s", mimeType)
if (reqPropsObj):
logger.debug("Request Properties Object given - using this")
reqPropsObjLoc = reqPropsObj
else:
logger.debug("No Request Properties Object given - creating one")
reqPropsObjLoc = ngamsArchiveUtils.ngamsReqProps.ngamsReqProps()
stagingFile = filename
try:
if (mimeType == None):
mimeType = ngamsHighLevelLib.determineMimeType(
srvObj.getCfg(), filename)
archive_start = time.time()
# Prepare dummy ngamsReqProps object (if an object was not given).
if (not reqPropsObj):
reqPropsObjLoc.setMimeType(mimeType)
reqPropsObjLoc.setStagingFilename(filename)
reqPropsObjLoc.setHttpMethod(NGAMS_HTTP_GET)
reqPropsObjLoc.setCmd(NGAMS_ARCHIVE_CMD)
reqPropsObjLoc.setSize(os.path.getsize(filename))
reqPropsObjLoc.setFileUri(NGAMS_HTTP_FILE_URL + filename)
reqPropsObjLoc.setNoReplication(noReplication)
# If no target disk is defined, find one suitable disk.
if (not reqPropsObjLoc.getTargDiskInfo()):
try:
trgDiskInfo = ngamsArchiveUtils.ngamsDiskUtils.\
findTargetDisk(srvObj.getHostId(),
srvObj.getDb(), srvObj.getCfg(),
mimeType, 0,
reqSpace=reqPropsObjLoc.getSize())
reqPropsObjLoc.setTargDiskInfo(trgDiskInfo)
# copy the file to the staging area of the target disk
stagingFile = trgDiskInfo.getMountPoint(
) + '/staging/' + os.path.basename(filename)
cpFile(filename, stagingFile)
reqPropsObjLoc.setStagingFilename(stagingFile)
except Exception, e:
errMsg = str(e) + ". Attempting to archive local file: " +\
filename
ngamsPlugInApi.notify(srvObj, NGAMS_NOTIF_NO_DISKS,
"NO DISKS AVAILABLE", errMsg)
raise Exception, errMsg
# Set the log cache to 1 during the handling of the file.
plugIn = srvObj.getMimeTypeDic()[mimeType]
logger.debug("Invoking DAPI: %s to handle file: %s", plugIn,
stagingFile)
plugInMethod = loadPlugInEntryPoint(plugIn)
resMain = plugInMethod(srvObj, reqPropsObjLoc)
# Move the file to final destination.
st = time.time()
mvFile(reqPropsObjLoc.getStagingFilename(),
resMain.getCompleteFilename())
iorate = reqPropsObjLoc.getSize() / (time.time() - st)
ngamsArchiveUtils.postFileRecepHandling(srvObj, reqPropsObjLoc,
resMain, trgDiskInfo)
def __handleCmd(srvObj, reqPropsObj):
"""
Handle the Mirroring Archive (MIRRARCHIVE) Command.
srvObj: Reference to NG/AMS server class object (ngamsServer).setState
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
Returns: Void.
"""
TRACE()
# Is this NG/AMS permitted to handle Archive Requests?
logger.debug("Checking if this NG/AMS permitted to handle Archive Requests?")
if (not srvObj.getCfg().getAllowArchiveReq()):
errMsg = genLog("NGAMS_ER_ILL_REQ", ["Archive"])
raise Exception(errMsg)
# Generate staging filename.
stgFilename = reqPropsObj.getStagingFilename()
logger.info("staging filename is: %s", stgFilename)
startByte = 0
if (os.path.exists(stgFilename) == 0):
logger.debug('this is a new staging file')
else:
startByte = os.path.getsize(stgFilename)
logger.debug('staging file already exists, requesting resumption of download from byte %d', startByte)
# Set reference in request handle object to the read socket.
try:
# Retrieve file_id and file_version from request proposal
file_id = reqPropsObj.fileinfo['fileId']
file_version = reqPropsObj.fileinfo['fileVersion']
logger.debug("Got file_id=%s and file_version=%s", file_id, file_version)
# Retrieve file contents (from URL, archive pull, or by storing the body
# of the HTTP request, archive push).
logger.info("Saving in staging file: %s", stgFilename)
stagingInfo = saveInStagingFile(srvObj, srvObj.getCfg(), reqPropsObj,
stgFilename, startByte)
reqPropsObj.incIoTime(stagingInfo[0])
checksumPlugIn = "ngamsGenCrc32"
checksum = stagingInfo[1]
except (ngamsFailedDownloadException.FailedDownloadException, ngamsFailedDownloadException.PostponeException):
raise
except Exception as e:
if getattr(e, 'errno', 0) == 28:
# we can't resume, otherwise the same host will be used next time
logger.warning("ran out of disk space during the download to %s. Marking as FAILURE", stgFilename)
# TBD automatically mark the volume as completed
# TBD try something more sophisticated with the file - other volumes on the same host?
# ot at least ARCHIVE in another host avoiding the download of WAN again. This is
# particularly important if we just downloaded most of a 300GB file
raise ngamsFailedDownloadException.FailedDownloadException(e)
elif reqPropsObj.getBytesReceived() >= 0:
logger.warning("the fetch has already downloaded data. marking as TORESUME")
raise ngamsFailedDownloadException.PostponeException(e)
else:
logger.warning(3, "no data has been downloaded yet. Marking as FAILURE")
raise ngamsFailedDownloadException.FailedDownloadException(e)
# Invoke DAPI
logger.info("Invoking DAPI")
resDapi = ngamsDAPIMirroring.ngamsGeneric(srvObj, reqPropsObj)
# Move file to final destination.
logger.info("Moving file to final destination: %s", resDapi.getCompleteFilename())
ioTime = mvFile(reqPropsObj.getStagingFilename(), resDapi.getCompleteFilename())
reqPropsObj.incIoTime(ioTime)
# Check/generate remaining file info + update in DB.
logger.info("Creating db entry")
creDate = srvObj.getDb().convertTimeStamp(getFileCreationTime(resDapi.getCompleteFilename()))
sqlUpdate = "update ngas_disks set available_mb = available_mb - {0} / (1024 * 1024), bytes_stored = bytes_stored + {1} " +\
"where disk_id = {2}"
srvObj.getDb().query2(sqlUpdate, args=(resDapi.getFileSize(), resDapi.getFileSize(), resDapi.getDiskId()))
ts = srvObj.getDb().convertTimeStamp(time.time())
sqlQuery = "INSERT INTO ngas_files " +\
"(disk_id, file_name, file_id, file_version, " +\
"format, file_size, " +\
"uncompressed_file_size, compression, " +\
"ingestion_date, %s, checksum, " % ('file_ignore' if srvObj.getCfg().getDbUseFileIgnore() else 'ignore') +\
"checksum_plugin, file_status, creation_date) "+\
"VALUES " +\
"({}, {}, {}, {}," +\
" {}, {}," +\
" {}, {}," +\
" {}, 0, {}," +\
" {}, {}, {})"
args = (str(resDapi.getDiskId()), str(resDapi.getRelFilename()), file_id, file_version,
str(resDapi.getFormat()), resDapi.getFileSize(),
resDapi.getUncomprSize(), str(resDapi.getCompression()),
ts, str(checksum),
checksumPlugIn, NGAMS_FILE_STATUS_OK, creDate)
logger.info("Will try to insert the file information: %s / %r", sqlQuery, args)
srvObj.getDb().query2(sqlQuery, args=args)
# Final log message
logger.info("Successfully handled Archive Pull Request for data file with URI: %s",
reqPropsObj.getSafeFileUri())
return
# EOF
def handleCmd(srvObj, reqPropsObj, httpRef):
"""
Handle the Proxy Quick Archive (PROXYQARCHIVE) Command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
T = TRACE()
# Check if the URI is correctly set.
logger.debug("Check if the URI is correctly set.")
if (reqPropsObj.getFileUri() == ""):
errMsg = genLog("NGAMS_ER_MISSING_URI")
raise Exception(errMsg)
#path = reqPropsObj.getHttpHdr('path')
if (not reqPropsObj.hasHttpPar('nexturl')):
errMsg = "Paremeter 'nexturl' is missing."
raise Exception(errMsg)
# Get mime-type (try to guess if not provided as an HTTP parameter).
logger.debug(
"Get mime-type (try to guess if not provided as an HTTP parameter).")
if (reqPropsObj.getMimeType() == ""):
mimeType = ngamsHighLevelLib.\
determineMimeType(srvObj.getCfg(), reqPropsObj.getFileUri())
reqPropsObj.setMimeType(mimeType)
## Set reference in request handle object to the read socket.
logger.debug("Set reference in request handle object to the read socket.")
rfile = httpRef.rfile
if reqPropsObj.getFileUri().startswith('http://'):
readFd = urllib.urlopen(reqPropsObj.getFileUri())
rfile = readFd
logger.debug("Generate basename filename from URI: %s",
reqPropsObj.getFileUri())
if (reqPropsObj.getFileUri().find("file_id=") >= 0):
file_id = reqPropsObj.getFileUri().split("file_id=")[1]
baseName = os.path.basename(file_id)
else:
baseName = os.path.basename(reqPropsObj.getFileUri())
blockSize = srvObj.getCfg().getBlockSize()
jobManHost = srvObj.getCfg().getNGASJobMANHost()
doCRC = srvObj.getCfg().getProxyCRC()
if (jobManHost):
saveFromHttpToHttp(reqPropsObj,
baseName,
blockSize,
rfile,
reportHost=jobManHost,
checkCRC=doCRC)
else:
saveFromHttpToHttp(reqPropsObj,
baseName,
blockSize,
rfile,
checkCRC=doCRC)
# Request after-math ...
srvObj.setSubState(NGAMS_IDLE_SUBSTATE)
msg = "Successfully handled Proxy (Quick) Archive Pull Request for data file " +\
"with URI: " + reqPropsObj.getSafeFileUri()
logger.info(msg)
return msg
def handleCmd(srvObj, reqPropsObj, httpRef):
"""
Handle the Quick Archive (QARCHIVE) Command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: (fileId, filePath) tuple.
"""
T = TRACE()
# Check if the URI is correctly set.
logger.debug("Check if the URI is correctly set.")
logger.debug("ReqPropsObj status: %s", reqPropsObj.getObjStatus())
parsDic = reqPropsObj.getHttpParsDic()
if (not parsDic.has_key('fileUri') or parsDic['fileUri'] == ""):
errMsg = genLog("NGAMS_ER_MISSING_URI")
raise Exception(errMsg)
else:
reqPropsObj.setFileUri(parsDic['fileUri'])
fileUri = reqPropsObj.getFileUri()
# Is this NG/AMS permitted to handle Archive Requests?
logger.debug("Is this NG/AMS permitted to handle Archive Requests?")
if (not srvObj.getCfg().getAllowArchiveReq()):
errMsg = genLog("NGAMS_ER_ILL_REQ", ["Archive"])
raise Exception, errMsg
srvObj.checkSetState("Archive Request", [NGAMS_ONLINE_STATE],
[NGAMS_IDLE_SUBSTATE, NGAMS_BUSY_SUBSTATE],
NGAMS_ONLINE_STATE,
NGAMS_BUSY_SUBSTATE,
updateDb=False)
# Get mime-type (try to guess if not provided as an HTTP parameter).
logger.debug(
"Get mime-type (try to guess if not provided as an HTTP parameter).")
if (not parsDic.has_key('mimeType') or parsDic['mimeType'] == ""):
mimeType = ""
reqPropsObj.setMimeType("")
else:
reqPropsObj.setMimeType(parsDic['mimeType'])
mimeType = reqPropsObj.getMimeType()
if (reqPropsObj.getMimeType() == ""):
mimeType = ngamsHighLevelLib.\
determineMimeType(srvObj.getCfg(), reqPropsObj.getFileUri())
reqPropsObj.setMimeType(mimeType)
else:
mimeType = reqPropsObj.getMimeType()
ioTime = 0
reqPropsObj.incIoTime(ioTime)
(resDapi, targDiskInfo, iorate) = archiveFromFile(srvObj, fileUri, 0,
mimeType, reqPropsObj)
if (resDapi == NGAMS_FAILURE):
errMsg = targDiskInfo
httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=500)
return
# Get crc info
# info(3, "Get checksum info")
# crc = None
# TODO: Investigate how CRC calculation could be performed best.
# checksumPlugIn = "ngamsGenCrc32"
# checksum = str(crc)
# info(3, "Invoked Checksum Plug-In: " + checksumPlugIn +\
# " to handle file: " + resDapi.getCompleteFilename() +\
# ". Result: " + checksum)
# Inform the caching service about the new file.
logger.debug("Inform the caching service about the new file.")
if (srvObj.getCachingActive()):
diskId = resDapi.getDiskId()
fileId = resDapi.getFileId()
fileVersion = 1
filename = resDapi.getRelFilename()
ngamsCacheControlThread.addEntryNewFilesDbm(srvObj, diskId, fileId,
fileVersion, filename)
# Update disk info in NGAS Disks.
logger.debug("Update disk info in NGAS Disks.")
srvObj.getDb().updateDiskInfo(resDapi.getFileSize(), resDapi.getDiskId())
# Check if the disk is completed.
# We use an approximate estimate for the remaning disk space to avoid
# to read the DB.
logger.debug("Check available space in disk")
availSpace = getDiskSpaceAvail(targDiskInfo.getMountPoint(), smart=False)
if (availSpace < srvObj.getCfg().getFreeSpaceDiskChangeMb()):
targDiskInfo.setCompleted(1).setCompletionDate(time.time())
targDiskInfo.write(srvObj.getDb())
# Request after-math ...
srvObj.setSubState(NGAMS_IDLE_SUBSTATE)
msg = "Successfully handled Archive Pull Request for data file " +\
"with URI: " + reqPropsObj.getSafeFileUri()
logger.info(msg)
httpRef.send_ingest_status(msg, targDiskInfo)
# Trigger Subscription Thread. This is a special version for MWA, in which we simply swapped MIRRARCHIVE and QARCHIVE
# [email protected]
logger.debug("triggering SubscriptionThread for file %s",
resDapi.getFileId())
srvObj.addSubscriptionInfo(
[(resDapi.getFileId(), resDapi.getFileVersion())], [])
srvObj.triggerSubscriptionThread()
return (resDapi.getFileId(),
'%s/%s' % (targDiskInfo.getMountPoint(), resDapi.getRelFilename()),
iorate)
def saveFromHttpToHttp(reqPropsObj,
basename,
blockSize,
rfile,
reportHost=None,
checkCRC=0):
"""
Save the data available on an HTTP channel into the given file.
ngamsCfgObj: NG/AMS Configuration object (ngamsConfig).
reqPropsObj: NG/AMS Request Properties object (ngamsReqProps).
basename: filename that will be put into HTTP content disposition (string).
blockSize: Block size (bytes) to apply when reading the data
from the HTTP channel (integer).
Returns: Tuple. Element 0: Time in took to write
file (s) (tuple).
"""
T = TRACE()
mimeType = reqPropsObj.getMimeType()
nexturl = reqPropsObj.getHttpPar('nexturl')
if (reqPropsObj.hasHttpPar('reporturl')):
rpurl = reqPropsObj.getHttpPar('reporturl')
else:
rpurl = None
#path = reqPropsObj.getHttpHdr('path')
#nexturl = path.split('=')[1]
contDisp = "attachment; filename=\"" + basename + "\""
contDisp += "; no_versioning=1"
logger.debug("Transferring data to : %s", nexturl)
start = time.time()
http = None
try:
# Distinguish between Archive Pull and Push Request. By Archive
# Pull we may simply read the file descriptor until it returns "".
sizeKnown = 0
if (reqPropsObj.is_GET()
and not reqPropsObj.getFileUri().startswith('http://')):
# (reqPropsObj.getSize() == -1)):
# Just specify something huge.
logger.debug(
"It is an Archive Pull Request/data with unknown size")
remSize = int(1e11)
elif reqPropsObj.getFileUri().startswith('http://'):
logger.debug(
"It is an HTTP Archive Pull Request: trying to get Content-Length"
)
httpInfo = rfile.info()
headers = httpInfo.headers
hdrsDict = ngamsLib.httpMsgObj2Dic(''.join(headers))
if hdrsDict.has_key('content-length'):
remSize = int(hdrsDict['content-length'])
else:
logger.debug("No HTTP header parameter Content-Length!")
logger.debug("Header keys: %s", hdrsDict.keys())
remSize = int(1e11)
else:
remSize = reqPropsObj.getSize()
logger.debug("Archive Push/Pull Request - Data size: %d", remSize)
sizeKnown = 1
http = buildHttpClient(
nexturl,
mimeType,
contDisp,
remSize,
checksum=reqPropsObj.getHttpHdr(NGAMS_HTTP_HDR_CHECKSUM))
# Receive the data.
buf = "-"
rdSize = blockSize
slow = blockSize / (512 * 1024.) # limit for 'slow' transfers
# sizeAccu = 0
lastRecepTime = time.time()
rdtt = 0 # total read time
wdtt = 0 # total write time
nb = 0 # number of blocks
srb = 0 # number of slow read blocks
swb = 0 # number of slow write blocks
tot_size = 0 # total number of bytes
nfailread = 0
crc = 0 # initialize CRC value
while ((remSize > 0) and ((time.time() - lastRecepTime) < 30.0)):
if (remSize < rdSize): rdSize = remSize
rdt = time.time()
buf = rfile.read(rdSize)
rdt = time.time() - rdt
if (rdt > slow):
srb += 1
rdtt += rdt
sizeRead = len(buf)
remSize -= sizeRead
tot_size += sizeRead
if (sizeRead > 0):
if (checkCRC):
crc = binascii.crc32(buf, crc)
wdt = time.time()
http._conn.sock.sendall(buf)
wdt = time.time() - wdt
wdtt += wdt
if wdt >= slow: swb += 1
nb += 1
lastRecepTime = time.time()
else:
logger.debug(
"Unsuccessful read attempt from HTTP stream! Sleeping 50 ms"
)
nfailread += 1
time.sleep(0.050)
deltaTime = time.time() - start
reqPropsObj.setBytesReceived(tot_size)
logger.debug("Data sent")
logger.debug(
"Receiving transfer time: %.3f s; Sending transfer time %.3f s",
rdtt, wdtt)
msg = "Sent data in file: %s. Bytes received / sent: %d. Time: %.3f s. " +\
"Rate: %.2f Bytes/s"
logger.debug(msg, basename, int(reqPropsObj.getBytesReceived()),
deltaTime,
(float(reqPropsObj.getBytesReceived()) / deltaTime))
# Raise a special info message if the transfer speed to disk or over network was
# slower than 512 kB/s
if srb > 0:
logger.warning(
"Number of slow network reads during this transfer: %d out of %d blocks. \
Consider checking the upstream network link!", srb, nb)
if swb > 0:
logger.warning(
"Number of slow network sends during this transfer: %d out of %d blocks. \
Consider checking your downstream network link!", swb, nb)
if nfailread > 0:
logger.warning(
"Number of failed reads during this transfer: %d out of %d blocks. \
Consider checking your upstream network!", nfailread, nb)
# Raise exception if less byes were received as expected.
if (sizeKnown and (remSize > 0)):
msg = genLog("NGAMS_ER_ARCH_RECV", [
reqPropsObj.getFileUri(),
reqPropsObj.getSize(), (reqPropsObj.getSize() - remSize)
])
raise Exception, msg
if (checkCRC):
checksum = reqPropsObj.getHttpHdr(NGAMS_HTTP_HDR_CHECKSUM)
if (checksum):
if (checksum != str(crc)):
msg = 'Checksum error for file %s, proxy crc = %s, but remote crc = %s' % (
reqPropsObj.getFileUri(), str(crc), checksum)
raise Exception(msg)
else:
logger.debug("%s CRC checked, OK!",
reqPropsObj.getFileUri())
processHttpReply(http, basename, nexturl)
except Exception, err:
if (str(err).find('Connection refused') > -1):
# The host on the nexturl is probably down
logger.exception("Fail to connect to the nexturl '%s'", nexturl)
# report this incident if the reporturl is available
if (rpurl):
logger.debug('Reporing this error to %s', rpurl)
urlreq = '%s?errorurl=%s&file_id=%s' % (rpurl, nexturl,
basename)
try:
urllib2.urlopen(urlreq)
except Exception:
logger.exception(
"Cannot report the error of nexturl '%s' to reporturl '%s' either",
nexturl, rpurl)
if (reportHost):
try:
rereply = urllib2.urlopen(
'http://%s/report/hostdown?file_id=%s&next_url=%s' %
(reportHost, basename, urllib2.quote(nexturl)),
timeout=15).read()
logger.info(
'Reply from sending file %s host-down event to server %s - %s',
basename, reportHost, rereply)
except Exception:
logger.exception(
'Fail to send host-down event to server %s',
reportHost)
raise err
def ngasExtractFitsHdrDppi(srvObj, reqPropsObj, filename):
"""
This DPPI extracts the main header from a FITS file
requested from the ESO Archive.
srvObj: Reference to instance of the NG/AMS Server
class (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
filename: Name of file to process (string).
Returns: DPPI return status object (ngamsDppiStatus).
Side effect: This DPPI works directly on the archived file, since it is
read-only access.
SPECIFIC DOCUMENTATION:
This DPPI controls the call to the printhead module. If the
Example URL (single line):
http://ngasdev1:7777/RETRIEVE
?file_id=MIDI.2004-02-11T04:16:04.528&
processing=ngasExtractFitsHdrDppi&
processing_pars=header%3D99
The header parameter is optional, if not specified the primary header
is returned.
Valid values for the header parameter are numbers between 0 and 99. If
numbers are specified which are either outside the range or bigger than
the number of headers (including the primary) the primary header will be
returned. Headers are counted from 0 starting with the primary header. 99
is a special value as it returns all headers concatenated in a single file.
If 'xml=vo' is specified headers are returned using a slightly modified
VOTable (XML) format. If 'xml=xfits' is specified headers are returned
using the XFits (XML) format. struct=1 returns the structure of the FITS
file. tsv=1 returns the headers in a tab separated format suitable for
direct ingest into the header repository.
"""
logger.debug("Entering ngasExtractFitsHdrDppi() ...")
T = TRACE()
statusObj = ngamsDppiStatus.ngamsDppiStatus()
if (reqPropsObj.hasHttpPar("processing_pars")):
pars = ngamsPlugInApi.\
parseRawPlugInPars(reqPropsObj.getHttpPar("processing_pars"))
else:
# default is to extract the primary header
pars = {'header': 0}
logger.debug("ngasExtractFitsHdrDppi: %s %r", filename, pars)
PARS = set(['header', 'xml', 'skey', 'struct', 'tsv', 'check'])
# initial settings for printhead
xtract = 0
parse = 0
xmlVals = ['xfits', 'vo']
xmlfl = ''
skeyfl = 0
skey = 'END'
show = 0
struct = 0
tsv = 0
check = 0
mergefl = 0
hfl = 0
mode = 1
result = ''
err = ''
ext = 'hdr'
if pars.has_key('header'):
# extract a certain header: if value == 99 all headers are extracted,
# for any other value that header is extracted. headers are
# counted from 0
hfl = 1
struct = 0
show = pars['header']
try:
head = int(show)
except:
err = "ngasExtractFitsHdrDppi: Invalid type for header " +\
"parameter. Should be int"
if head < 0 or head > 99:
err = "ngasExtractFitsHdrDppi: Invalid value specified for " +\
"header parameter."
if pars.has_key('xml'):
# if this key exists we do a conversion to XFits XML.
struct = 0
if pars['xml'] in xmlVals:
xmlfl = pars['xml']
else:
err = "ngasExtractFitsHdrDppi: Invalid value for xml " +\
"parameter. Should be 'vo|xfits': "+ pars['xml']
ext = 'xml'
if pars.has_key('skey'):
# extract just one keyword. CAUTION: No checking done!
skey = pars['skey'].strip()
skeyfl = 1
keyParts = skey.split()
ext = 'txt'
head = int(head)
if head < 0: head = 0
if ((not re.match('[a-zA-Z]', skey[0]))
or (len(keyParts) > 1 and keyParts[0] != 'HIERARCH')
or (len(keyParts[0]) > 8)):
err = "ngasExtractFitsHdrDppi: Invalid value for skey " +\
"parameter specified. Must be a valid FITS keyword:",\
skey
if pars.has_key('struct'):
# return only the structure of the FITS file. Value of the
# parameter is ignored
head = -99
struct = 1
ext = 'txt'
if pars.has_key('tsv'):
# extract header in tsv format. Parameter value is ignored
struct = 1
tsv = 1
ext = 'txt'
head = int(head)
if head < 0: head = 0
if pars.has_key('check'):
# head structure and calculate the checksum of the data part.
head = -99
struct = 1
check = 1
# printhead supports a list of files, but here we only use one
fils = [filename]
base = os.path.basename(filename)
pos = base.rfind('.fits')
file_id = base[:pos]
if len(set(pars) - PARS) != 0: # detect unsupported parameters
xpars = set(pars) - PARS
err = "ngasExtractFitsHdrDppi: Unsupported option(s): %s\n" + \
"Valid options are:\n" + \
"header=<number> where number is an integer between 0 " +\
"and max(extension)-1 or 99 (for all)\n" +\
"xml=<format>, where format is [vo|xfits]\n" +\
"struct=1\n" +\
"skey=<FITS keyword>, should be a valid keyword, crude " +\
"checking is done\n" + \
"tsv=1\n\n" + \
"combinations are allowed and should be separated by " +\
"a comma character.\n"
err = err % xpars
ext = 'txt'
file_id = 'error'
result = '' # initialize result string
if err != '':
result = err
if result == '':
for f in fils:
cmd = constructCommand(f, head, struct, skey, tsv, xmlfl, mode,
check)
logger.debug('Executing command: %s', cmd)
stat, result, _ = execCmd(cmd)
if stat != 0:
errMsg = "Processing of header for file %s failed: %s" % (
filename, result)
raise Exception(errMsg)
resFilename = file_id + "." + ext
try:
# mime-type guessing does not work sometimes, we force it in that case.
mimeType = ngamsPlugInApi.determineMimeType(srvObj.getCfg(),
resFilename)
except:
if ext == 'xml':
mimeType = 'text/xml'
else:
mimeType = 'text/ascii'
resObj = ngamsDppiStatus.ngamsDppiResult(NGAMS_PROC_DATA, mimeType, result,
resFilename, '')
statusObj.addResult(resObj)
logger.debug("Leaving ngasExtractFitsHdrDppi() ...")
return statusObj
def delSubscriber(srvObj,
subscrId):
"""
Remove a Susbcriber from the internal list + from the DB.
srvObj: Reference to NG/AMS Server object (ngamsServer).
subscrId: Subscriber ID (string).
Returns: Void.
"""
T = TRACE()
err = 0
errMsg = ''
try:
srvObj.getDb().deleteSubscriber(subscrId)
except Exception:
estr = " Error deleting Subscriber information from the DB. " +\
"Subscriber ID: %s"
logger.exception(estr, str(subscrId))
err += 1
errMsg += estr
# remove all entries associated with this subscriber from in-memory dictionaries
numThreads = 0
if (srvObj.getSubscriberDic().has_key(subscrId)):
subscriber = srvObj.getSubscriberDic()[subscrId]
numThreads = subscriber.getConcurrentThreads()
del srvObj.getSubscriberDic()[subscrId]
else:
estr = " Cannot find Subscriber with an ID '%s' kept internally. " % subscrId
err += 1
errMsg += estr
if (srvObj._subscrScheduledStatus.has_key(subscrId)):
del srvObj._subscrScheduledStatus[subscrId]
else:
estr = " Cannot find scheduled status for the subscriber '%s' kept internally. " % subscrId
err += 1
errMsg += estr
if (srvObj._subscrSuspendDic.has_key(subscrId)):
srvObj._subscrSuspendDic[subscrId].set() # resume all suspended deliveryThreads (if any) so they can know the subscriber is removed
del srvObj._subscrDeliveryThreadDic[subscrId] # this does not kill those deliveryThreads, but only the list container
else:
estr = " Cannot find delivery threads for the subscriber '%s' kept internally. " % subscrId
err += 1
errMsg += estr
deliveryThreadRefDic = srvObj._subscrDeliveryThreadDicRef
deliveryFileDic = srvObj._subscrDeliveryFileDic
for tid in range(int(numThreads)):
thrdName = NGAMS_DELIVERY_THR + subscrId + str(tid)
if (deliveryThreadRefDic.has_key(thrdName)):
del deliveryThreadRefDic[thrdName]
if (deliveryFileDic.has_key(thrdName)):
del deliveryFileDic[thrdName]
fileDeliveryCountDic = srvObj._subscrFileCountDic
fileDeliveryCountDic_Sem = srvObj._subscrFileCountDic_Sem
# reduce the file reference count by 1 for all files that are in the queue to be delivered
# and in the meantime, clear the queue before deleting it
if (srvObj._subscrQueueDic.has_key(subscrId)):
if (srvObj.getCachingActive()):
errOld = err
qu = srvObj._subscrQueueDic[subscrId]
while (1):
fileinfo = None
try:
fileinfo = qu.get_nowait()
except Queue.Empty, e:
break
if (fileinfo is None):
break
#fileInfo = ngamsSubscriptionThread._convertFileInfo(fileinfo)
fileId = fileinfo[ngamsSubscriptionThread.FILE_ID]
fileVersion = fileinfo[ngamsSubscriptionThread.FILE_VER]
err += _reduceRefCount(fileDeliveryCountDic, fileDeliveryCountDic_Sem, fileId, fileVersion)
if ((err - errOld) > 0):
errMsg += ' Error reducing file reference count for some files in the queue, check NGAS log to find out which files'
del srvObj._subscrQueueDic[subscrId]
def locateArchiveFile(srvObj,
fileId,
fileVersion=-1,
diskId="",
hostId="",
reqPropsObj=None):
"""
Locate the file indicated by the File ID. Returns a list containing
the necessary information for retrieving the file:
[<Location>, <File Host>, <IP Address>, <Port No>, <Mount Point>,
<Filename>, <File ID>, <File Version>, <Mime-Type>]
- whereby:
<Location> = Location of the file (NGAMS_HOST_LOCAL,
NGAMS_HOST_CLUSTER, NGAMS_HOST_DOMAIN,
NGAMS_HOST_REMOTE).
<File Host> = Host ID of host to be contacted to get access to the
file.
<IP Address> = IP Address of host to be contacted to get access to the
file.
<Port No> = Port number used by the NG/AMS Server.
<Mount Point> = Mount point at which the file is residing.
<Filename> = Name of file relative to mount point.
<File ID> = ID of file.
<File Version> = Version of file.
<Mime-Type> = Mime-type of file (as registered in NGAS).
srvObj: Reference to NG/AMS server class object (ngamsServer).
fileId: File ID of file to locate (string).
fileVersion: Version of the file (integer).
diskId: ID of the disk hosting the file (string).
hostId: ID of the host where the file is located (string).
reqPropsObj: Request Property object to keep track of actions done during
the request handling (ngamsReqProps|None).
Returns: List with information about file location (list).
"""
T = TRACE()
# Get a list with the candidate files matching the query conditions.
res = srvObj.getDb().getFileInfoFromFileId(fileId,
fileVersion,
diskId,
ignore=0,
dbCursor=False)
# r[-2] is the host_id, r[-1] is the mount point
all_info = []
for r in res:
file_info = ngamsFileInfo.ngamsFileInfo().unpackSqlResult(r)
all_info.append((file_info, r[-2], r[-1]))
return _locateArchiveFile(srvObj, fileId, fileVersion, diskId, hostId,
reqPropsObj, all_info)
def handleCmd(srvObj, reqPropsObj, httpRef):
"""
Handle Command QUERY to query the DB system used.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
T = TRACE()
# Get command parameters.
query = None
args = ()
if (reqPropsObj.hasHttpPar("query")):
query = reqPropsObj.getHttpPar("query")
qkey = query
if query.lower() in valid_queries.keys():
query = valid_queries[query.lower()]
else:
msg = "Invalid query specified. Valid queries are: %s" %\
valid_queries.keys()
raise Exception, msg
if reqPropsObj.getHttpPar(
"query") == 'subscribers_like' or reqPropsObj.getHttpPar(
"query") == 'files_like' or reqPropsObj.getHttpPar(
"query") == 'files_location' or reqPropsObj.getHttpPar(
"query") == 'lastver_location':
param = '%'
if (reqPropsObj.hasHttpPar("like")):
param = reqPropsObj.getHttpPar("like")
args = (param, )
if reqPropsObj.getHttpPar("query") == 'files_between':
param1 = param2 = ''
if (reqPropsObj.hasHttpPar("start")):
param1 = reqPropsObj.getHttpPar("start")
if (reqPropsObj.hasHttpPar("end")):
param2 = reqPropsObj.getHttpPar("end")
if param1 and param2:
args = (param1, param2)
elif param1:
query = 'select * from ngas_files where ingestion_date >= {0}'
args = (param1, )
else:
query = valid_queries['files_list']
else:
msg = "No query specified. Valid queries are: %s" %\
valid_queries.keys()
raise Exception, msg
out_format = None
if (reqPropsObj.hasHttpPar("format")):
out_format = reqPropsObj.getHttpPar("format")
cursorId = None
if (reqPropsObj.hasHttpPar("cursor_id")):
cursorId = reqPropsObj.getHttpPar("cursor_id")
fetch = None
if (reqPropsObj.hasHttpPar("fetch")):
fetch = int(reqPropsObj.getHttpPar("fetch"))
# Execute the query.
if (not cursorId):
res = srvObj.getDb().query2(query, args=args)
# TODO: Make possible to return an XML document
# TODO: Potential problem with very large result sets.
# Implement streaming result directly.
if (out_format == "list"):
header = None
if reqPropsObj.getHttpPar("query") not in [
'files_stats', 'files_list_recent', 'files_location',
'lastver_location'
]:
if query.find('ngas_files') >= 0:
header = NGAMS_FILES_COLS
elif query.find('ngas_disks') >= 0:
header = NGAMS_DISKS_COLS
elif query.find('ngas_subscribers') >= 0:
header = NGAMS_SUBSCR_COLS
elif reqPropsObj.getHttpPar("query") == 'files_stats':
header = ['Number of files', 'Total volume [MB]']
elif reqPropsObj.getHttpPar("query") == 'files_list_recent':
header = [
'file_id', 'file_name', 'file_size', 'ingestion_date'
]
elif reqPropsObj.getHttpPar("query") == 'files_location':
header = [
'host_id', 'file_full_path', 'file_version',
'ingestion_date'
]
elif reqPropsObj.getHttpPar("query") == 'lastver_location':
header = [
'host_id', 'file_full_path', 'file_version',
'creation_date', 'ingestion_date'
]
finalRes = formatAsList(res, header=header)
"""
if query.find('ngas_files') >=0:
header = NGAMS_FILES_COLS
elif query.find('ngas_disks') >= 0:
header = NGAMS_DISKS_COLS
else:
header = None
finalRes = formatAsList(res, header=header)
"""
mimeType = NGAMS_TEXT_MT
elif (out_format == "pickle"):
finalRes = cPickle.dumps([res])
mimeType = NGAMS_PYTHON_PICKLE_MT
elif (out_format == "json"):
jsobj = createJsonObj(res, qkey)
finalRes = json.dumps(jsobj, default=encode_decimal)
mimeType = NGAMS_JSON_MT
else:
finalRes = str([res])
mimeType = NGAMS_PYTHON_LIST_MT
# Return the data.
httpRef.send_data(finalRes, mimeType)
elif (fetch):
cursorDbmFilename = genCursorDbmName(srvObj.getCfg().\
getRootDirectory(), cursorId)
if (not os.path.exists(cursorDbmFilename)):
logger.error("Illegal Cursor ID: %s or cursor expired", cursorId)
return []
try:
cursorDbm = ngamsDbm.ngamsDbm(cursorDbmFilename, writePerm=1)
count = 0
resSet = []
cursorIdx = cursorDbm.get(CURSOR_IDX)
while (count < fetch):
cursorIdx += 1
if (cursorDbm.hasKey(str(cursorIdx))):
res = cursorDbm.get(str(cursorIdx))
resSet.append(res)
else:
# -- no more results to return.
break
count += 1
cursorDbm.add(CURSOR_IDX, cursorIdx)
cursorDbm.sync()
del cursorDbm
# If all entries have been fetched, we delete the cursor DBM.
if (count < fetch):
rmFile(cursorDbmFilename + "*")
# Return the data.
# TODO: Make it possible to return ASCII List + XML.
httpRef.send_data(str(resSet), NGAMS_PYTHON_LIST_MT)
except Exception, e:
msg = "Error fetching from cursor with ID: %s. Error: %s"
raise Exception, msg % (cursorId, str(e))
def _locateArchiveFile(srvObj, fileId, fileVersion, diskId, hostId,
reqPropsObj, files):
"""
See description of ngamsFileUtils.locateArchiveFile(). This function is
used simply to encapsulate the complete processing to be able to clean up.
"""
T = TRACE()
msg = "_locateArchiveFile() - Disk ID: %s - File ID: " +\
"%s - File Version: %d ..."
logger.debug(msg, str(diskId), fileId, int(fileVersion))
# Filter out files not on specified host if host ID is given.
if (hostId):
files = filter(lambda x: x[1] == hostId, files)
# If no file was found we raise an exception.
if not files:
tmpFileRef = fileId
if (fileVersion > 0): tmpFileRef += "/Version: " + str(fileVersion)
if (diskId): tmpFileRef += "/Disk ID: " + diskId
if (hostId): tmpFileRef += "/Host ID: " + hostId
errMsg = genLog("NGAMS_ER_UNAVAIL_FILE", [tmpFileRef])
raise Exception(errMsg)
# We now sort the file information sub-lists in the file list.
# The priori is as follows:
#
# 1. Local host.
# 2. Same cluster.
# 3. Same domain (e.g. hq.eso.org).
# 4. Other files (remote files).
localHostFileList = []
clusterFileList = []
domainFileList = []
remoteFileList = []
all_hosts = set([x[1] for x in files])
hostDic = ngamsHighLevelLib.resolveHostAddress(srvObj.getHostId(),
srvObj.getDb(),
srvObj.getCfg(), all_hosts)
# Loop over the candidate files and sort them.
fileCount = idx = 0
for fileInfo in files:
fileHost = fileInfo[1]
if (hostDic[fileHost].getHostType() == NGAMS_HOST_LOCAL):
localHostFileList.append(fileInfo)
elif (hostDic[fileHost].getHostType() == NGAMS_HOST_CLUSTER):
clusterFileList.append(fileInfo)
elif (hostDic[fileHost].getHostType() == NGAMS_HOST_DOMAIN):
domainFileList.append(fileInfo)
else:
# NGAMS_HOST_REMOTE:
remoteFileList.append(fileInfo)
idx += 1
fileCount += 1
# The highest priority of the file is determined by the File Version,
# the latest version is preferred even though this may be stored
# on another NGAS host.
# A dictionary is built up, which contains the candidate files. The
# format is such there each version found is one key. For each key
# (= version) there is a list with the corresponding file information
# order according to the location.
candFileDic = {}
fileLists = [[NGAMS_HOST_LOCAL, localHostFileList],
[NGAMS_HOST_CLUSTER, clusterFileList],
[NGAMS_HOST_DOMAIN, domainFileList],
[NGAMS_HOST_REMOTE, remoteFileList]]
for fileListInfo in fileLists:
location = fileListInfo[0]
fileList = fileListInfo[1]
for fileInfo in fileList:
fileVer = fileInfo[0].getFileVersion()
# Create a list in connection with each File Version key.
if fileVer not in candFileDic:
candFileDic[fileVer] = []
candFileDic[fileVer].append([location, fileInfo[0], fileInfo[1]])
fileVerList = list(candFileDic)
fileVerList.sort(reverse=True)
if logger.level <= logging.DEBUG:
msg = ""
count = 1
for fileVer in fileVerList:
for fi in candFileDic[fileVer]:
msg += "(" + str(count) + ": Location:" + fi[0] +\
", Host:" + fi[2] + ", Version:" +\
str(fi[1].getFileVersion()) + ") "
count += 1
logger.debug("File list to check: " + msg)
# If no files were found we raise an exception.
if (len(candFileDic) == 0):
if (fileVersion != -1):
fileRef = fileId + "/V" + str(fileVersion)
else:
fileRef = fileId
errMsg = genLog("NGAMS_ER_UNAVAIL_FILE", [fileRef])
raise Exception(errMsg)
# We generate a list with the Disk IDs (which we need later).
# Generate a dictionary with Disk Info Objects.
diskIdDic = {}
for fileVer in fileVerList:
for fileInfo in candFileDic[fileVer]:
diskIdDic[fileInfo[1].getDiskId()] = fileInfo[1].getDiskId()
sqlDiskInfo = srvObj.getDb().getDiskInfoFromDiskIdList(list(diskIdDic))
diskInfoDic = {}
for diskInfo in sqlDiskInfo:
diskInfoObj = ngamsDiskInfo.ngamsDiskInfo().unpackSqlResult(diskInfo)
diskInfoDic[diskInfoObj.getDiskId()] = diskInfoObj
logger.debug("Disk Info Objects Dictionary: %s", str(diskInfoDic))
# Check if the files are accessible - when the first accessible file
# in the fileList is found, the information is returned as the file wanted.
# To check the file accessibility, it is also checked if the NG/AMS
# 'responsible' for the file, allows for Retrieve Requests (only done
# in connection with a Retrieve Request).
logger.debug(
"Checking which of the candidate files should be selected ...")
foundFile = 0
for fileVer in fileVerList:
if (foundFile): break
for fileInfo in candFileDic[fileVer]:
location = fileInfo[0]
fileInfoObj = fileInfo[1]
host = fileInfo[2]
diskInfoObj = diskInfoDic[fileInfoObj.getDiskId()]
port = hostDic[host].getSrvPort()
logger.debug("Checking candidate file with ID: %s on host/port: %s/%s. " + \
"Location: %s",
fileInfoObj.getFileId(), host, str(port), location)
# If the file is stored locally we check if it is accessible,
# otherwise we send a STATUS/file_access request to the
# host in question.
if (location == NGAMS_HOST_LOCAL):
# Check first if the local system supports retrieve requests.
# (if relevant).
if (reqPropsObj):
if (reqPropsObj.getCmd() == NGAMS_RETRIEVE_CMD):
if (not srvObj.getCfg().getAllowRetrieveReq()):
continue
# Check if the file is accessible.
filename = os.path.normpath(diskInfoObj.getMountPoint()+"/" +\
fileInfoObj.getFilename())
logger.debug(
"Checking if local file with name: %s is available",
filename)
if (not os.path.exists(filename)):
logger.debug(
genLog("NGAMS_INFO_FILE_NOT_AVAIL", [fileId, host]))
else:
logger.debug(
genLog("NGAMS_INFO_FILE_AVAIL", [fileId, host]))
foundFile = 1
break
else:
logger.debug("Checking if file with ID/Version: %s/%s " +\
"is available on host/port: %s/%s",
fileInfoObj.getFileId(), str(fileInfoObj.getFileVersion()),
host, str(port))
# If a server hosting a file is suspended, it is woken up
# to be able to check if the file is really accessible.
if (hostDic[host].getSrvSuspended() == 1):
logger.debug("Server hosting requested file (%s/%s) is suspended " + \
"- waking up server ...",
host, str(port))
try:
ngamsSrvUtils.wakeUpHost(srvObj, host)
logger.debug("Suspended server hosting requested file (%s/%s) " +\
"has been woken up",
host, str(port))
except Exception:
logger.exception("Error waking up server hosting selected " +\
"file")
continue
# The file is hosted on a host, which is not suspended or
# which was successfully woken up.
pars = [["file_access", fileInfoObj.getFileId()]]
if (fileInfoObj.getFileVersion() != -1):
pars.append(["file_version", fileInfoObj.getFileVersion()])
ipAddress = hostDic[host].getIpAddress()
authHdr = ngamsSrvUtils.genIntAuthHdr(srvObj)
resp = ngamsHttpUtils.httpGet(ipAddress,
port,
NGAMS_STATUS_CMD,
pars=pars,
auth=authHdr)
with contextlib.closing(resp):
data = resp.read()
statusObj = ngamsStatus.ngamsStatus().unpackXmlDoc(data, 1)
if logger.isEnabledFor(logging.DEBUG):
logger.debug(
"Result of File Access Query: %s",
re.sub("\n", "",
str(statusObj.genXml().toprettyxml(' ',
'\n'))))
if ((statusObj.getMessage().\
find("NGAMS_INFO_FILE_AVAIL") == -1)):
logger.debug(
genLog("NGAMS_INFO_FILE_NOT_AVAIL", [fileId, host]))
else:
logger.debug(
genLog("NGAMS_INFO_FILE_AVAIL", [fileId, host]))
foundFile = 1
break
# If no file was found we raise an exception.
if (not foundFile):
errMsg = genLog("NGAMS_ER_UNAVAIL_FILE", [fileId])
raise Exception(errMsg)
# The file was found, get the info necessary for the acquiring the file.
ipAddress = hostDic[host].getIpAddress()
srcFileInfo = [
location, host, ipAddress, port,
diskInfoObj.getMountPoint(),
fileInfoObj.getFilename(),
fileInfoObj.getFileId(),
fileInfoObj.getFileVersion(),
fileInfoObj.getFormat()
]
msg = "Located suitable file for request - File ID: %s. " +\
"Info for file found - Location: %s - Host ID/IP: %s/%s - " +\
"Port Number: %s - File Version: %d - Filename: %s - " +\
"Mime-type: %s"
logger.debug(msg, fileId, location, host, ipAddress, port,
fileInfoObj.getFileVersion(), fileInfoObj.getFilename(),
fileInfoObj.getFormat())
return srcFileInfo
def ngamsLinuxOnlinePlugIn(srvObj, reqPropsObj=None):
"""
Function mounts all NGAMS disks and loads the kernel module for the IDE
controller card. It returns the NGAMS specific disk info dictionary.
srvObj: Reference to instance of the NG/AMS Server
class (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: Disk info dictionary (dictionary).
"""
T = TRACE()
rootMtPt = srvObj.getCfg().getRootDirectory()
parDic = ngamsPlugInApi.\
parseRawPlugInPars(srvObj.getCfg().getOnlinePlugInPars())
if (parDic.has_key("module")):
stat = ngamsLinuxSystemPlugInApi.insMod(parDic["module"])
else:
stat = 0
if (stat == 0):
if (parDic.has_key("module")):
msg = "Kernel module " + parDic["module"] + " loaded"
logger.info(msg)
# Old format = unfortunately some Disk IDs of WDC/Maxtor were
# generated wrongly due to a mistake by IBM, which lead to a wrong
# implementation of the generation of the Disk ID.
if (not parDic.has_key("old_format")):
oldFormat = 0
raise Warning, "Missing Online Plug-In Parameter: old_format=0|1"
else:
oldFormat = int(parDic["old_format"])
# The controllers Plug-In Parameter, specifies the number of controller
# in the system.
if (not parDic.has_key("controllers")):
controllers = None
else:
controllers = parDic["controllers"]
# Get start index for the 3ware disk devices.
if (not parDic.has_key("dev_start_idx")):
devStartIdx = "a"
else:
devStartIdx = parDic["dev_start_idx"]
# AWI: added this to fix problem at the ATF
# Get start index for NGAS disk devices
if (not parDic.has_key("ngas_start_idx")):
ngasStartIdx = devStartIdx
else:
ngasStartIdx = parDic["ngas_start_idx"]
# Try first to umount possibly mounted disks (forced).
ngamsLinuxSystemPlugInApi.umount(rootMtPt)
# Select between 3ware WEB Interface and 3ware Command Line Tool.
if (parDic["uri"].find("http") != -1):
diskDic = ngamsEscaladeUtils.parseHtmlInfo(parDic["uri"], rootMtPt)
else:
diskDic = ngamsEscaladeUtils.\
parseCmdLineInfo(rootMtPt,
controllers,
oldFormat,
slotIds = ["*"],
devStartIdx = devStartIdx,
ngasStartIdx = ngasStartIdx)
#####ngamsLinuxSystemPlugInApi.removeFstabEntries(diskDic)
ngamsLinuxSystemPlugInApi.ngamsMount(srvObj, diskDic,
srvObj.getCfg().getSlotIds())
return diskDic
else:
errMsg = "Problem executing ngamsLinuxOnlinePlugIn"
errMsg = genLog("NGAMS_ER_ONLINE_PLUGIN", [errMsg])
raise Exception(errMsg)
def handleCmd(srvObj,
reqPropsObj,
httpRef):
"""
Handle STATUS command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler object
(ngamsHttpRequestHandler).
Returns: Void.
"""
T = TRACE()
status = ngamsStatus.ngamsStatus()
status.\
setDate(toiso8601()).\
setVersion(getNgamsVersion()).setHostId(srvObj.getHostId()).\
setStatus(NGAMS_SUCCESS).\
setMessage("Successfully handled command STATUS").\
setState(srvObj.getState()).setSubState(srvObj.getSubState())
reqPropsObjRef = reqPropsObj
# Get the information requested.
diskId = ""
fileId = ""
fileVersion = ""
configurationFile = ""
fileAccess = ""
hostId = ""
requestId = ""
dbTime = ""
dbTimeReset = ""
fileList = ""
fileListId = ""
maxElements = 100000
if (reqPropsObj.hasHttpPar("disk_id")):
diskId = reqPropsObj.getHttpPar("disk_id")
if (reqPropsObj.hasHttpPar("file_id")):
fileId = reqPropsObj.getHttpPar("file_id")
if (reqPropsObj.hasHttpPar("file_version")):
fileVersion = reqPropsObj.getHttpPar("file_version")
if (reqPropsObj.hasHttpPar("configuration_file")): configurationFile = "-"
if (reqPropsObj.hasHttpPar("file_access")):
fileAccess = reqPropsObj.getHttpPar("file_access")
if (reqPropsObj.hasHttpPar("host_id")):
hostId = reqPropsObj.getHttpPar("host_id")
if (reqPropsObj.hasHttpPar("max_elements")):
maxElements = int(reqPropsObj.getHttpPar("max_elements"))
if (reqPropsObj.hasHttpPar("request_id")):
requestId = reqPropsObj.getHttpPar("request_id")
if (reqPropsObj.hasHttpPar("db_time")):
dbTime = True
if (reqPropsObj.hasHttpPar("db_time_reset")):
dbTimeReset = True
if (reqPropsObj.hasHttpPar("flush_log")):
# in the past this called flushLog()
# do we really want to keep that functionality? I doubt it
pass
if (reqPropsObj.hasHttpPar("file_list")):
fileList = int(reqPropsObj.getHttpPar("file_list"))
if (reqPropsObj.hasHttpPar("file_list_id")):
fileListId = reqPropsObj.getHttpPar("file_list_id")
if (reqPropsObj.hasHttpPar("dump_object_info")):
# Dump status of all objects allocated into file specified.
targFile = reqPropsObj.getHttpPar("dump_object_info")
rmFile(targFile)
fo = open(targFile, "w")
fo.write(_genObjectStatus())
fo.close()
# Handle request for file info.
genCfgStatus = 0
genDiskStatus = 0
genFileStatus = 0
genStatesStatus = 1
genRequestStatus = 0
msg = ""
help = 0
if (reqPropsObj.hasHttpPar("help")):
global _help
msg = _help
help = 1
elif hostId and hostId != srvObj.getHostId():
host, port = srvObj.get_remote_server_endpoint(hostId)
cfgObj = srvObj.getCfg()
if not cfgObj.getProxyMode():
httpRef.redirect(host, port)
return
else:
try:
httpRef.proxy_request(hostId, host, port)
except Exception, e:
ex = re.sub("<|>", "", str(e))
errMsg = genLog("NGAMS_ER_COM",
[host, port,ex])
raise Exception, errMsg
return
def ngamsMWAOnlinePlugIn(srvObj, reqPropsObj=None):
"""
Scan the specified Root Directory/Volume Directory for NGAS Volumes
and register these as operational volumes.
srvObj: Reference to instance of the NG/AMS Server
class (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: Disk info dictionary (dictionary).
"""
T = TRACE()
rootDir = srvObj.getCfg().getRootDirectory()
volumeDir = srvObj.getCfg().getVolumeDirectory()
# Build the root directory for the NGAS Volumes and make a glob to
# get directories under this.
ngasVolDir = os.path.normpath(rootDir + os.sep + volumeDir)
dirList = glob.glob(ngasVolDir + os.sep + "*")
diskInfoDic = {}
for dir in dirList:
# Check if a '.ngas_volume_id' is found under the directory.
volInfoFile = os.path.\
normpath(dir + os.sep + NGAS_VOL_INFO_FILE)
if (os.path.exists(volInfoFile)):
# - It exists, load it
volInfoDic = loadVolInfoFile(volInfoFile)
# Create an ngamsPhysDiskInfo object with the information about
# the slot.
diskId = volInfoDic[NGAS_VOL_INFO_ID]
devName = NGAS_VOL_INFO_IGNORE
portNo = -1
slotId = dir.split(os.sep)[-1]
mtPt = dir
status = "OK"
capGb = -1
model = NGAS_VOL_INFO_IGNORE
serialNo = NGAS_VOL_INFO_IGNORE
if (volInfoDic.has_key(NGAS_VOL_INFO_TYPE)):
diskType = volInfoDic[NGAS_VOL_INFO_TYPE]
else:
diskType = NGAS_VOL_INFO_IGNORE
if (volInfoDic.has_key(NGAS_VOL_INFO_MANUFACT)):
manufact = volInfoDic[NGAS_VOL_INFO_MANUFACT]
else:
manufact = NGAS_VOL_INFO_IGNORE
msg = "Registering volume with parameters: Disk ID: %s, " +\
"Device: %s, Port No: %s, Slot ID: %s, Mount Point: %s, "+\
"Status: %s, Capacity (GB): %s, Model: %s, Serial#: %s, " +\
"Type: %s, Manufacturer: %s"
logger.debug(msg,
diskId, devName, str(portNo), slotId, mtPt, status,
str(capGb), model, serialNo, diskType, manufact)
diskInfoDic[str(slotId)] = ngamsPhysDiskInfo.\
ngamsPhysDiskInfo().\
setPortNo(portNo).\
setSlotId(slotId).\
setMountPoint(mtPt).\
setStatus(status).\
setCapacityGb(capGb).\
setModel(model).\
setSerialNo(serialNo).\
setType(diskType).\
setManufacturer(manufact).\
setDiskId(diskId).\
setDeviceName(devName)
notifyRegistrationService(srvObj)
#cmdMod = "ngamsCmd_ASYNCLISTRETRIEVE"
#srvObj.getDynCmdDic()[cmdMod] = 1
#host = getHostName()
#port = srvObj.getCfg().getPortNo()
#startAsyncRetrListUrl = "http://" + host + ":" + str(port) + "/ASYNCLISTRETRIEVE?ngassystem=start"
#info(3, "Sending system starting request ")
#myRes = ngamsCmd_ASYNCLISTRETRIEVE.startAsyncQService(srvObj, reqPropsObj)
#strRes = urllib.urlopen(startAsyncRetrListUrl).read()
#myRes = pickle.loads(strRes)
#info(3, "Starting async retrieve list result - %s" % myRes)
#_restoreSubscriptionInfoFromDisk(srvObj)
return diskInfoDic
def handleCmd(srvObj, reqPropsObj, httpRef):
"""
Handle STATUS command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler object
(ngamsHttpRequestHandler).
Returns: Void.
"""
T = TRACE()
status = ngamsStatus.ngamsStatus()
status.\
setDate(toiso8601()).\
setVersion(getNgamsVersion()).setHostId(srvObj.getHostId()).\
setStatus(NGAMS_SUCCESS).\
setMessage("Successfully handled command STATUS").\
setState(srvObj.getState()).setSubState(srvObj.getSubState())
reqPropsObjRef = reqPropsObj
# Get the information requested.
diskId = ""
fileId = ""
fileVersion = ""
configurationFile = ""
fileAccess = ""
hostId = ""
requestId = ""
dbTime = ""
dbTimeReset = ""
fileList = ""
fileListId = ""
maxElements = 100000
if (reqPropsObj.hasHttpPar("disk_id")):
diskId = reqPropsObj.getHttpPar("disk_id")
if (reqPropsObj.hasHttpPar("file_id")):
fileId = reqPropsObj.getHttpPar("file_id")
if (reqPropsObj.hasHttpPar("file_version")):
fileVersion = int(reqPropsObj.getHttpPar("file_version"))
if (reqPropsObj.hasHttpPar("configuration_file")): configurationFile = "-"
if (reqPropsObj.hasHttpPar("file_access")):
fileAccess = reqPropsObj.getHttpPar("file_access")
if (reqPropsObj.hasHttpPar("host_id")):
hostId = reqPropsObj.getHttpPar("host_id")
if (reqPropsObj.hasHttpPar("max_elements")):
maxElements = int(reqPropsObj.getHttpPar("max_elements"))
if (reqPropsObj.hasHttpPar("request_id")):
requestId = reqPropsObj.getHttpPar("request_id")
if (reqPropsObj.hasHttpPar("db_time")):
dbTime = True
if (reqPropsObj.hasHttpPar("db_time_reset")):
dbTimeReset = True
if (reqPropsObj.hasHttpPar("flush_log")):
# in the past this called flushLog()
# do we really want to keep that functionality? I doubt it
pass
if (reqPropsObj.hasHttpPar("file_list")):
fileList = int(reqPropsObj.getHttpPar("file_list"))
if (reqPropsObj.hasHttpPar("file_list_id")):
fileListId = reqPropsObj.getHttpPar("file_list_id")
if (reqPropsObj.hasHttpPar("dump_object_info")):
# Dump status of all objects allocated into file specified.
targFile = reqPropsObj.getHttpPar("dump_object_info")
rmFile(targFile)
fo = open(targFile, "w")
fo.write(_genObjectStatus())
fo.close()
# Handle request for file info.
genCfgStatus = 0
genDiskStatus = 0
genFileStatus = 0
genStatesStatus = 1
genRequestStatus = 0
msg = ""
help = 0
if (reqPropsObj.hasHttpPar("help")):
global _help
msg = _help
help = 1
elif hostId and hostId != srvObj.getHostId():
host, port = srvObj.get_remote_server_endpoint(hostId)
cfgObj = srvObj.getCfg()
if not cfgObj.getProxyMode():
httpRef.redirect(host, port)
return
else:
try:
httpRef.proxy_request(hostId, host, port)
except Exception as e:
ex = re.sub("<|>", "", str(e))
errMsg = genLog("NGAMS_ER_COM", [host, port, ex])
raise Exception(errMsg)
return
elif (fileList):
if (not fileListId):
# It's a new STATUS?file_list request.
fileListId = _handleFileList(srvObj, reqPropsObj, httpRef)
# Send back data from the request.
_handleFileListReply(srvObj, reqPropsObj, httpRef, fileListId,
maxElements)
elif (diskId):
diskObj = ngamsDiskInfo.ngamsDiskInfo()
diskObj.read(srvObj.getDb(), diskId)
status.addDiskStatus(diskObj)
genDiskStatus = 1
elif (fileId):
if (not fileVersion): fileVersion = -1
fileObj = ngamsFileInfo.ngamsFileInfo()
fileObj.read(srvObj.getHostId(), srvObj.getDb(), fileId, fileVersion)
diskObj = ngamsDiskInfo.ngamsDiskInfo()
try:
diskObj.read(srvObj.getDb(), fileObj.getDiskId())
except:
errMsg = "Illegal Disk ID found: %s for file with ID: %s" %\
(fileObj.getDiskId(), fileId)
raise Exception(errMsg)
diskObj.addFileObj(fileObj)
status.addDiskStatus(diskObj)
genDiskStatus = 1
genFileStatus = 1
elif (requestId):
logger.debug("Checking status of request with ID: %s", requestId)
reqPropsObjRef = srvObj.getRequest(requestId)
if (not reqPropsObjRef):
errMsg = genLog("NGAMS_ER_ILL_REQ_ID", [requestId])
raise Exception(errMsg)
genRequestStatus = 1
elif (configurationFile):
msg = "configuration_file=" + srvObj.getCfg().getCfg()
genCfgStatus = 1
status.setNgamsCfgObj(srvObj.getCfg())
elif (fileAccess):
if (not fileVersion): fileVersion = -1
fileId = fileAccess
msg = _checkFileAccess(srvObj, reqPropsObj, httpRef, fileId,
fileVersion, diskId)
elif (dbTime):
logger.debug("Querying total DB time")
msg = "Total DB time: %.6fs" % srvObj.getDb().getDbTime()
elif (dbTimeReset):
msg = "Resetting DB timer"
logger.debug(msg)
srvObj.getDb().resetDbTime()
else:
msg = "Successfully handled command STATUS"
if (reqPropsObjRef == reqPropsObj):
reqPropsObj.setCompletionTime()
srvObj.updateRequestDb(reqPropsObj)
if (genCfgStatus or genDiskStatus or genFileStatus or genRequestStatus):
status.setReqStatFromReqPropsObj(reqPropsObjRef)
# Generate XML reply.
xmlStat = status.genXmlDoc(genCfgStatus, genDiskStatus, genFileStatus,
genStatesStatus)
xmlStat = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, xmlStat)
httpRef.send_data(six.b(xmlStat), NGAMS_XML_MT)
elif not httpRef.reply_sent:
httpRef.send_status(msg)
if (msg and (not help)):
logger.info(msg)
else:
logger.info("Successfully handled command STATUS")
def ngamsCheckfileCacheCtrlPI(srvObj, cacheEntryObj):
"""
Cache Control Plug-In, which checks for the availability of a file in
one or more NGAS Archives. If a valid copy of the file is found in the
specified locations, the file is signalled for deletion.
srvObj: Reference to NG/AMS Server Object (ngamsServer).
cacheEntryObj: Cache Entry Object containing the information for the
cached file (ngamsCacheEntry).
Returns: Returns True if the file can be deleted from the cache,
otherwise False (boolean).
"""
T = TRACE()
try:
_unpackCheckPlugInPars(srvObj)
# Should the check be done?
global CheckfileCacheCtrlPI_checkingPeriod
if (CheckfileCacheCtrlPI_checkingPeriod):
lastCheck = cacheEntryObj.getPar(LAST_CHECK)
if (lastCheck):
# Check if it is at least checking-period seconds since the
# last check was done. If this is the case, the check is not
# yet due.
timeNow = time.time()
if ((timeNow - lastCheck) <
CheckfileCacheCtrlPI_checkingPeriod):
return False
else:
# First time the plug-in is invoked on this file. Just set the
# time to the current time, then the file will be checked
# approx. after checking-period seconds.
cacheEntryObj.addPar(LAST_CHECK, time.time())
return False
# Loop over the given lists.
global CheckfileCacheCtrlPI_sets
okCount = 0
for set in CheckfileCacheCtrlPI_sets:
if (_fileAvailableInSet(set, cacheEntryObj)):
okCount += 1
else:
# No need to check further ...
break
# Update the checking time.
cacheEntryObj.addPar(LAST_CHECK, time.time())
# Check the result.
if (okCount == len(CheckfileCacheCtrlPI_sets)):
# File was available in at least one valid copy in all systems.
return True
else:
# File was not available in at least one valid copy in all sets.
return False
except Exception, e:
msg = "Error ocurred executing Cache Control Plug-In. Error: %s" %\
str(e)
logger.error(msg)
raise
def ngamsGenDapi(srvObj,
reqPropsObj):
"""
Generic Data Archiving Plug-In to handle archiving of any file.
srvObj: Reference to NG/AMS Server Object (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: Standard NG/AMS Data Archiving Plug-In Status
as generated by: ngamsPlugInApi.genDapiSuccessStat()
(ngamsDapiStatus).
"""
T = TRACE()
# For now the exception handling is pretty basic:
# If something goes wrong during the handling it is tried to
# move the temporary file to the Bad Files Area of the disk.
logger.debug("Plug-In handling data for file: %s",
os.path.basename(reqPropsObj.getFileUri()))
try:
parDic = {}
handlePars(reqPropsObj, parDic)
diskInfo = reqPropsObj.getTargDiskInfo()
stgFile = reqPropsObj.getStagingFilename()
ext = os.path.splitext(stgFile)[1][1:]
# Generate file information.
logger.debug("Generate file information")
dateDir = toiso8601(fmt=FMT_DATE_ONLY)
fileVersion, relPath, relFilename,\
complFilename, fileExists =\
ngamsPlugInApi.genFileInfo(srvObj.getDb(),
srvObj.getCfg(),
reqPropsObj, diskInfo,
reqPropsObj.\
getStagingFilename(),
parDic[FILE_ID],
parDic[FILE_ID], [dateDir])
complFilename, relFilename = checkForDblExt(complFilename,
relFilename)
# Compress the file if requested.
uncomprSize, archFileSize, format, compression, comprExt =\
compressFile(srvObj, reqPropsObj, parDic)
if (comprExt != ""):
complFilename += ".%s" % comprExt
relFilename += ".%s" % comprExt
logger.debug("DAPI finished processing file - returning to host application")
return ngamsPlugInApi.genDapiSuccessStat(diskInfo.getDiskId(),
relFilename,
parDic[FILE_ID],
fileVersion, format,
archFileSize, uncomprSize,
compression, relPath,
diskInfo.getSlotId(),
fileExists, complFilename)
except Exception as e:
msg = "Error occurred in DAPI: %s" % str(e)
logger.error(msg)
raise Exception(genLog("NGAMS_ER_DAPI_RM", [msg]))
def remFile(srvObj, reqPropsObj, diskId, fileId, fileVersion, execute):
"""
Function used to delete information about files in the NGAS DB and to
delete files on the disks. Only local files are considered.
The information selected for deletion is selected applying the
following criterias:
o diskId!="", fileId=="", fileVersion==-1:
All files on the given disk are taken.
o diskId!="", fileId!="", fileVersion==-1:
All files with the given File ID on the disk with the given ID
will be selected. No specific file version will be taken into account.
o diskId!="", fileId!="", fileVersion!=-1:
The referenced file with the given File ID and File Version on the
given ID is selected (if this exists).
o diskId=="", fileId!="", fileVersion==-1:
Not allowed.
o diskId=="", fileId!="", fileVersion!=-1:
Not allowed.
o diskId=="", fileId=="", fileVersion!=-1:
Not allowed.
o diskId!="", fileId=="", fileVersion!=-1:
Not allowed
The function will not carry out the actual deletion of the selected
files if the 'execute' parameter is set to 0. In this case a report
with the items selected for deletion will be generated.
NOTE: No files are selected for deletion, unless there are at least three
copies of the file (File ID + File Version), which apparently are
accessible. In addition, these three copies have to be distributed
on three different disks (storage media).
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
diskId: ID of disk. It is not possible to specify a pattern,
must be exact ID (string).
fileId: ID of file. It is possible to specify a file pattern
using wild cards (string).
fileVersion: Version of file to delete. If not specified, all
files matching the Disk ID + File ID will be selected.
Otherwise it is possible to specify a single file for
deletion (integer).
execute: If set to 0, only a report of the information that has
been selected for deletion is generated (integer/0|1).
Returns: Status object with a list of disks and corresponding
files deleted (ngamsStatus).
"""
T = TRACE()
tmpFilePat = ngamsHighLevelLib.genTmpFilename(srvObj.getCfg(),
"REMFILE_CMD")
try:
status = _remFile(srvObj, reqPropsObj, diskId, fileId, fileVersion,
execute, tmpFilePat)
return status
finally:
rmFile(tmpFilePat + "*")
def handleCmd(srvObj,
reqPropsObj,
httpRef):
"""
Handle DISCARD Command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
T = TRACE()
if (reqPropsObj.hasHttpPar("help")):
global _help
return _help
if (not srvObj.getCfg().getAllowRemoveReq()):
errMsg = genLog("NGAMS_ER_ILL_REQ", ["Discard File"])
raise Exception(errMsg)
diskId = None
fileId = None
fileVersion = None
hostId = None
path = None
execute = 0
if (reqPropsObj.hasHttpPar("disk_id")):
diskId = reqPropsObj.getHttpPar("disk_id")
if (reqPropsObj.hasHttpPar("file_id")):
fileId = reqPropsObj.getHttpPar("file_id")
if (reqPropsObj.hasHttpPar("file_version")):
try:
fileVersion = int(reqPropsObj.getHttpPar("file_version"))
except ValueError:
raise Exception("Illegal value for File Version specified: " +\
str(fileVersion))
if (reqPropsObj.hasHttpPar("host_id")):
hostId = reqPropsObj.getHttpPar("host_id")
if (reqPropsObj.hasHttpPar("path")):
path = reqPropsObj.getHttpPar("path")
if (reqPropsObj.hasHttpPar("execute")):
try:
execute = int(reqPropsObj.getHttpPar("execute"))
except:
errMsg = genLog("NGAMS_ER_REQ_HANDLING", ["Must provide proper " +\
"value for parameter: execute (0|1)"])
raise Exception(errMsg)
msg = _discardFile(srvObj, diskId, fileId, fileVersion, hostId, path, execute)
if msg.startswith("NGAMS_INFO_"):
status = NGAMS_SUCCESS
else:
status = NGAMS_FAILURE
httpRef.send_status(msg, status=status, code=NGAMS_HTTP_SUCCESS)
def handleCmd(srvObj, reqPropsObj, httpRef):
"""
Handle the CARCHIVE Command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: (fileId, filePath) tuple.
"""
T = TRACE()
# Is this NG/AMS permitted to handle Archive Requests?
logger.debug("Is this NG/AMS permitted to handle Archive Requests?")
if (not srvObj.getCfg().getAllowArchiveReq()):
errMsg = genLog("NGAMS_ER_ILL_REQ", ["Archive"])
raise Exception, errMsg
srvObj.checkSetState("Archive Request", [NGAMS_ONLINE_STATE],
[NGAMS_IDLE_SUBSTATE, NGAMS_BUSY_SUBSTATE],
NGAMS_ONLINE_STATE,
NGAMS_BUSY_SUBSTATE,
updateDb=False)
if httpRef.command != 'POST':
raise Exception("Only POST allowed for CARCHIVE")
# Get mime-type (try to guess if not provided as an HTTP parameter).
logger.debug(
"Get mime-type (try to guess if not provided as an HTTP parameter).")
if (reqPropsObj.getMimeType() == ""):
mimeType = ngamsHighLevelLib.\
determineMimeType(srvObj.getCfg(), reqPropsObj.getFileUri())
reqPropsObj.setMimeType(mimeType)
else:
mimeType = reqPropsObj.getMimeType()
# Determine the target volume, ignoring the stream concept.
logger.debug("Determine the target volume, ignoring the stream concept.")
targDiskInfo = ngamsArchiveUtils._random_target_volume(srvObj)
if (targDiskInfo == None):
errMsg = "No disk volumes are available for ingesting any files."
raise Exception(errMsg)
reqPropsObj.setTargDiskInfo(targDiskInfo)
# Generate staging filename.
logger.debug("Generate staging filename from URI: %s",
reqPropsObj.getFileUri())
if (reqPropsObj.getFileUri().find("file_id=") >= 0):
file_id = reqPropsObj.getFileUri().split("file_id=")[1]
baseName = os.path.basename(file_id)
else:
baseName = os.path.basename(reqPropsObj.getFileUri())
stgFilename = os.path.join("/", targDiskInfo.getMountPoint(),
NGAMS_STAGING_DIR,
genUniqueId() + "___" + baseName)
logger.debug("Staging filename is: %s", stgFilename)
reqPropsObj.setStagingFilename(stgFilename)
# Retrieve file contents (from URL, archive pull, or by storing the body
# of the HTTP request, archive push).
stagingInfo = saveInStagingFile(srvObj.getCfg(), reqPropsObj, httpRef,
stgFilename, targDiskInfo)
ioTime = stagingInfo[0]
rootContainer = stagingInfo[1]
fileDataList = stagingInfo[2]
ingestRate = stagingInfo[3]
reqPropsObj.incIoTime(ioTime)
createContainers(rootContainer, None, srvObj)
from ngamsLib import ngamsPlugInApi
import ngamsGenDapi
parDic = {}
ngamsGenDapi.handlePars(reqPropsObj, parDic)
diskInfo = reqPropsObj.getTargDiskInfo()
# Generate file information.
logger.debug("Generate file information")
dateDir = toiso8601(fmt=FMT_DATE_ONLY)
resDapiList = []
containerSizes = {}
for item in fileDataList:
container = item[0]
filepath = item[1]
crc = item[2]
containerId = str(container.getContainerId())
basename = os.path.basename(filepath)
fileId = basename
fileVersion, relPath, relFilename,\
complFilename, fileExists =\
ngamsPlugInApi.genFileInfo(srvObj.getDb(),
srvObj.getCfg(),
reqPropsObj, diskInfo,
filepath,
fileId,
basename, [dateDir])
complFilename, relFilename = ngamsGenDapi.checkForDblExt(
complFilename, relFilename)
# Keep track of the total size of the container
uncomprSize = ngamsPlugInApi.getFileSize(filepath)
if not containerSizes.has_key(containerId):
containerSizes[containerId] = 0
containerSizes[containerId] += uncomprSize
mimeType = reqPropsObj.getMimeType()
compression = "NONE"
archFileSize = ngamsPlugInApi.getFileSize(filepath)
resDapi = ngamsPlugInApi.genDapiSuccessStat(
diskInfo.getDiskId(), relFilename, fileId, fileVersion, mimeType,
archFileSize, uncomprSize, compression, relPath,
diskInfo.getSlotId(), fileExists, complFilename)
# Move file to final destination.
logger.debug("Moving file to final destination")
ioTime = mvFile(filepath, resDapi.getCompleteFilename())
reqPropsObj.incIoTime(ioTime)
# Get crc info
checksumPlugIn = "StreamCrc32"
checksum = str(crc)
# Get source file version
# e.g.: http://ngas03.hq.eso.org:7778/RETRIEVE?file_version=1&file_id=X90/X962a4/X1
logger.debug("Get file version")
file_version = resDapi.getFileVersion()
if reqPropsObj.getFileUri().count("file_version"):
file_version = int(
(reqPropsObj.getFileUri().split("file_version=")[1]
).split("&")[0])
# Check/generate remaining file info + update in DB.
logger.debug("Creating db entry")
creDate = getFileCreationTime(resDapi.getCompleteFilename())
fileInfo = ngamsFileInfo.ngamsFileInfo().\
setDiskId(resDapi.getDiskId()).\
setFilename(resDapi.getRelFilename()).\
setFileId(resDapi.getFileId()).\
setFileVersion(file_version).\
setFormat(resDapi.getFormat()).\
setFileSize(resDapi.getFileSize()).\
setUncompressedFileSize(resDapi.getUncomprSize()).\
setCompression(resDapi.getCompression()).\
setIngestionDate(time.time()).\
setChecksum(checksum).setChecksumPlugIn(checksumPlugIn).\
setFileStatus(NGAMS_FILE_STATUS_OK).\
setCreationDate(creDate).\
setIoTime(reqPropsObj.getIoTime())
fileInfo.write(srvObj.getHostId(), srvObj.getDb())
# Add the file to the container
srvObj.getDb().addFileToContainer(containerId, resDapi.getFileId(),
True)
# Update the container sizes
for contSizeInfo in containerSizes.iteritems():
srvObj.getDb().setContainerSize(contSizeInfo[0], contSizeInfo[1])
# Inform the caching service about the new file.
logger.debug("Inform the caching service about the new file.")
if (srvObj.getCachingActive()):
diskId = resDapi.getDiskId()
fileId = resDapi.getFileId()
fileVersion = file_version
filename = resDapi.getRelFilename()
ngamsCacheControlThread.addEntryNewFilesDbm(
srvObj, diskId, fileId, fileVersion, filename)
# Update disk info in NGAS Disks.
logger.debug("Update disk info in NGAS Disks.")
srvObj.getDb().updateDiskInfo(resDapi.getFileSize(),
resDapi.getDiskId())
resDapiList.append(resDapi)
# Check if the disk is completed.
# We use an approximate extimate for the remaning disk space to avoid
# to read the DB.
logger.debug("Check available space in disk")
availSpace = getDiskSpaceAvail(targDiskInfo.getMountPoint(), smart=False)
if (availSpace < srvObj.getCfg().getFreeSpaceDiskChangeMb()):
targDiskInfo.setCompleted(1).setCompletionDate(time.time())
targDiskInfo.write(srvObj.getDb())
# Request after-math ...
srvObj.setSubState(NGAMS_IDLE_SUBSTATE)
msg = "Successfully handled Archive Pull Request for data file " +\
"with URI: " + reqPropsObj.getSafeFileUri()
logger.info(msg)
httpRef.send_ingest_status(msg, targDiskInfo)
for resDapi in resDapiList:
# Trigger Subscription Thread. This is a special version for MWA, in which we simply swapped MIRRARCHIVE and QARCHIVE
# [email protected]
logger.debug("triggering SubscriptionThread for file %s",
resDapi.getFileId())
srvObj.addSubscriptionInfo(
[(resDapi.getFileId(), resDapi.getFileVersion())], [])
srvObj.triggerSubscriptionThread()
def _registerExec(srvObj,
fileListDbmName,
tmpFilePat,
diskInfoDic,
reqPropsObj = None):
"""
Register the files listed in the File List DBM (ngamsDbm), which
match the mime-type(s) either specified in the 'mimeType' parameter,
or if this is not specified, which match all the mime-types specified
in the configuration file.
When the registration procedure has been executed, the function sends
an Email Notification message indicating which files were registered
if the HTTP parameter 'notif_email' is given.
The functions creates a File Info Objects per file handled and
writes this in a temporary file, which is a DBM file. The
keys in this DB is simply the file number in the sequence of files
handled, pointing to a pickled ngamsFileInfo object.
Each of the File Info Objects indicates if the file was registered or not.
This is done by setting the tag of the File Info Object to one of the
following values:
REGISTERED: The file was successfully registered in the NGAS DB
FAILED: The file was selected for registration but could not
be properly registered because of inconsistencies.
The status will be of the format: 'FAILED[: <reason>]'.
REJECTED: A file found under the specified path directory was
not accepted for cloning, usually because the mime-type
was not correct. The status will be of the format:
'REJECTED[: <reason>]'.
Note, that registration is comparable to archiving of files. For that
reason a DAPI must be provided for each type of file that should be
registered. If this is not fullfilled, the file registration will
fail.
Only files stored on one of the NGAS disks configured in the
configuration file, are considered. Files stored in other locations
are ignored.
srvObj: Instance of NG/AMS Server object (ngamsServer).
fileListDbmName: Name of a DBM containing the information
about the files to be registered. Each element in the
list is referred to by a key, which is a number.
These points to a pickled list for each file containing
the following information:
[<Filename>, <Disk ID>, <Mime-Type>]
The information for each disk concerned is also contained
in the DB referred to by its Disk ID and by its mount
point. The data is a pickled instance of the ngamsDiskInfo
class (string).
tmpFilePat: Pattern for temporary files used during the registration
process (string).
diskInfoDic: Dictionary with Disk IDs as keys pointing to the info
about the disk (dictionary/ngamsDiskInfo).
reqPropsObj: If an NG/AMS Request Properties Object is given, the
Request Status will be updated as the request is carried
out (ngamsReqProps).
Returns: Void.
"""
T = TRACE()
emailNotif = 0
if (reqPropsObj):
if (reqPropsObj.hasHttpPar("notif_email")):
emailNotif = 1
# Create the temporary BSD DB to contain the information for the
# Email Notification Message.
if (emailNotif):
regDbmName = tmpFilePat + "_NOTIF_EMAIL"
regDbm = ngamsDbm.ngamsDbm(regDbmName, writePerm = 1)
# Open the DBM containing the list of files to (possibly) register.
fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm = 1)
# Want to parse files in alphabetical order.
# TODO: Portatibility issue. Try to avoid UNIX shell commands for sorting.
tmpFileList = tmpFilePat + "_FILE_LIST"
rmFile(tmpFileList)
fo = open(tmpFileList, "w")
fileListDbm.initKeyPtr()
while (1):
dbmKey, fileInfo = fileListDbm.getNext()
if (not dbmKey): break
fo.write(dbmKey + "\n")
fo.close()
sortFileList = tmpFilePat + "_SORT_FILE_LIST"
rmFile(sortFileList)
shellCmd = "sort %s > %s" % (tmpFileList, sortFileList)
stat, out = commands.getstatusoutput(shellCmd)
if (stat != 0):
raise Exception, "Error executing command: %s. Error: %s" %\
(shellCmd, str(out))
rmFile(tmpFileList)
# Go through each file in the list, check if the mime-type is among the
# ones, which apply for registration. If yes try to register the file
# by invoking the corresponding DAPI on the file.
checksumPlugIn = srvObj.getCfg().getChecksumPlugIn().strip()
fileRegCount = 0
fileFailCount = 0
fileRejectCount = 0
regTimeAccu = 0.0
fileCount = 0
fo = open(sortFileList)
run = 1
while (run):
reg_start = time.time()
dbmKey = fo.readline()
if (dbmKey.strip() == ""):
run = 0
continue
fileInfo = fileListDbm.get(dbmKey[0:-1])
filename = fileInfo[0]
diskId = fileInfo[1]
mimeType = fileInfo[2]
# Register the file. Check first, that exactly this file is
# not already registered. In case it is, the file will be rejected.
regPi = srvObj.getCfg().register_plugins[mimeType]
logger.debug("Plugin found for %s: %s", mimeType, regPi)
params = ngamsPlugInApi.parseRawPlugInPars(regPi.pars)
tmpReqPropsObj = ngamsReqProps.ngamsReqProps().\
setMimeType(mimeType).\
setStagingFilename(filename).\
setTargDiskInfo(diskInfoDic[diskId]).\
setHttpMethod(NGAMS_HTTP_GET).\
setCmd(NGAMS_REGISTER_CMD).\
setSize(os.path.getsize(filename)).\
setFileUri(filename).\
setNoReplication(1)
tmpFileObj = ngamsFileInfo.ngamsFileInfo()
try:
# Invoke Registration Plug-In.
piName = regPi.name
plugInMethod = loadPlugInEntryPoint(piName)
piRes = plugInMethod(srvObj, tmpReqPropsObj, params)
del tmpReqPropsObj
# Check if this file is already registered on this disk. In case
# yes, it is not registered again.
files = srvObj.db.getFileSummary1(srvObj.getHostId(), [piRes.getDiskId()],
[piRes.getFileId()])
fileRegistered = 0
for tmpFileInfo in files:
tmpMtPt = tmpFileInfo[ngamsDbCore.SUM1_MT_PT]
tmpFilename = tmpFileInfo[ngamsDbCore.SUM1_FILENAME]
tmpComplFilename = os.path.normpath(tmpMtPt + "/" +\
tmpFilename)
if (tmpComplFilename == filename):
fileRegistered = 1
break
if (fileRegistered):
fileRejectCount += 1
tmpMsgForm = "REJECTED: File with File ID/Version: %s/%d " +\
"and path: %s is already registered on disk " +\
"with Disk ID: %s"
tmpMsg = tmpMsgForm % (piRes.getFileId(),
piRes.getFileVersion(), filename,
piRes.getDiskId())
logger.warning(tmpMsg + ". File is not registered again.")
if (emailNotif):
tmpFileObj.\
setDiskId(diskId).setFilename(filename).\
setTag(tmpMsg)
regDbm.addIncKey(tmpFileObj)
if (reqPropsObj):
reqPropsObj.incActualCount(1)
ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj,
regTimeAccu)
regTimeAccu += time.time() - reg_start
fileCount += 1
continue
# Calculate checksum (if plug-in specified).
if (checksumPlugIn != ""):
logger.debug("Invoking Checksum Plug-In: %s to handle file: %s",
checksumPlugIn, filename)
plugInMethod = loadPlugInEntryPoint(checksumPlugIn)
checksum = plugInMethod(srvObj, filename, 0)
else:
checksum = ""
# Move file and update information about file in the NGAS DB.
mvFile(filename, piRes.getCompleteFilename())
ngamsArchiveUtils.updateFileInfoDb(srvObj, piRes, checksum,
checksumPlugIn)
ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), piRes)
ngamsLib.makeFileReadOnly(piRes.getCompleteFilename())
if (emailNotif):
uncomprSize = piRes.getUncomprSize()
ingestDate = time.time()
creDateSecs = getFileCreationTime(filename)
tmpFileObj.\
setDiskId(diskId).\
setFilename(filename).\
setFileId(piRes.getFileId()).\
setFileVersion(piRes.getFileVersion()).\
setFormat(piRes.getFormat()).\
setFileSize(piRes.getFileSize()).\
setUncompressedFileSize(uncomprSize).\
setCompression(piRes.getCompression()).\
setIngestionDate(ingestDate).\
setIgnore(0).\
setChecksum(checksum).\
setChecksumPlugIn(checksumPlugIn).\
setFileStatus(NGAMS_FILE_STATUS_OK).\
setCreationDate(creDateSecs).\
setTag("REGISTERED")
fileRegCount += 1
# If running as a cache archive, update the Cache New Files DBM
# with the information about the new file.
if (srvObj.getCachingActive()):
fileVer = fio.getFileVersion()
ngamsCacheControlThread.addEntryNewFilesDbm(srvObj, diskId,
piRes.getFileId(),
fileVer, filename)
# Generate a confirmation log entry.
msg = genLog("NGAMS_INFO_FILE_REGISTERED",
[filename, piRes.getFileId(), piRes.getFileVersion(),
piRes.getFormat()])
time.sleep(0.005)
regTime = time.time() - reg_start
msg = msg + ". Time: %.3fs." % (regTime)
logger.info(msg, extra={'to_syslog': 1})
except Exception, e:
errMsg = genLog("NGAMS_ER_FILE_REG_FAILED", [filename, str(e)])
logger.error(errMsg)
if (emailNotif):
tmpFileObj.\
setDiskId(diskId).setFilename(filename).\
setTag(errMsg)
fileFailCount += 1
regTime = time.time() - reg_start
# TODO (rtobar, 2016-01): Why don't we raise an exception here?
# Otherwise the command appears as successful on the
# client-side
# Add the file information in the registration report.
if (emailNotif): regDbm.addIncKey(tmpFileObj)
# Update request status time information.
regTimeAccu += regTime
if (reqPropsObj):
reqPropsObj.incActualCount(1)
ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj,
regTimeAccu)
fileCount += 1
def saveFromHttpToFile(ngamsCfgObj,
reqPropsObj,
httpRef,
trgFilename,
blockSize,
mutexDiskAccess=1,
diskInfoObj=None):
"""
Save the data available on an HTTP channel into the given file.
ngamsCfgObj: NG/AMS Configuration object (ngamsConfig).
reqPropsObj: NG/AMS Request Properties object (ngamsReqProps).
trgFilename: Target name for file where data will be
written (string).
blockSize: Block size (bytes) to apply when reading the data
from the HTTP channel (integer).
mutexDiskAccess: Require mutual exclusion for disk access (integer).
diskInfoObj: Disk info object. Only needed if mutual exclusion
is required for disk access (ngamsDiskInfo).
Returns: Tuple. Element 0: Time in took to write
file (s) (tuple).
"""
T = TRACE()
checkCreatePath(os.path.dirname(trgFilename))
start = time.time()
try:
# Make mutual exclusion on disk access (if requested).
if (mutexDiskAccess):
ngamsHighLevelLib.acquireDiskResource(ngamsCfgObj,
diskInfoObj.getSlotId())
# Distinguish between Archive Pull and Push Request. By Archive
# Pull we may simply read the file descriptor until it returns "".
remSize = reqPropsObj.getSize()
logger.debug("Archive Push/Pull Request - Data size: %d", remSize)
handler = ngamsMIMEMultipart.FilesystemWriterHandler(
blockSize, True, trgFilename)
parser = ngamsMIMEMultipart.MIMEMultipartParser(
handler, httpRef.rfile, remSize, blockSize)
parser.parse()
deltaTime = time.time() - start
fileDataList = handler.getFileDataList()
crcTime = handler.getCrcTime()
writingTime = handler.getWritingTime()
rootContainer = handler.getRoot()
readingTime = parser.getReadingTime()
bytesRead = parser.getBytesRead()
ingestRate = (float(bytesRead) / deltaTime)
reqPropsObj.setBytesReceived(bytesRead)
logger.debug(
"Transfer time: %.3f s; CRC time: %.3f s; write time %.3f s",
readingTime, crcTime, writingTime)
return [deltaTime, rootContainer, fileDataList, ingestRate]
finally:
# Release disk resouce.
if (mutexDiskAccess):
ngamsHighLevelLib.releaseDiskResource(ngamsCfgObj,
diskInfoObj.getSlotId())
def cloneCheckDiskSpace(srvObj,
cloneListDbmName,
tmpFilePat,
targetDiskId = ""):
"""
Check if there is enough disk space available on this NGAS host for
carrying out the Clone Request.
srvObj: Reference to instance of Server Object (ngamsServer).
cloneListDbmName: Name of DBM containing the information about
the files to be cloned. This DB has an index number
as key pointing to pickled information about each file.
This pickled information is
[<File Info Object>, <Host ID>, <Mount Point>]
(string)
tmpFilePat: File pattern to be used for generating temporary
files (string).
targetDiskId: ID of disk to where the files cloned should be
written (string).
Returns: Void.
"""
T = TRACE()
# Make a dictionary with the available amount of space per disk.
logger.debug("Generating dictionary with available space per disk")
availDiskSpaceDic = {}
mtRootDir = srvObj.getCfg().getRootDirectory()
if (targetDiskId):
tmpDiskInfoObjList = [ngamsDiskInfo.ngamsDiskInfo().\
read(srvObj.getDb(), targetDiskId)]
else:
tmpDiskInfoObjList = ngamsDiskUtils.\
getDiskInfoForMountedDisks(srvObj.getDb(),
srvObj.getHostId(),
mtRootDir)
for diskInfoObj in tmpDiskInfoObjList:
mtPt = diskInfoObj.getMountPoint()
diskId = diskInfoObj.getDiskId()
availDiskSpaceDic[diskId] = getDiskSpaceAvail(mtPt, format="B")
# Now simulate the execution of the clone job by going sequentially
# through the files selected for cloning and subtract the required
# disk space from the available amount for each disk.
#
# When a disk reaches the threshold for considering the disk as
# completed, it is removed from the list of available disks.
cloneListDbm = ngamsDbm.ngamsDbm(cloneListDbmName)
# This dictionary contains a list of files that have been simulated
# successfully cloned. Since they are not updated in the DB, we keep
# track of this locally.
cloneSucDbmName = tmpFilePat + "_CLONE_SUCCESS_DB"
rmFile(cloneSucDbmName + "*")
cloneSucDbm = ngamsDbm.ngamsDbm(cloneSucDbmName, cleanUpOnDestr=1,
writePerm=1)
# This is used to keep track of the files that could not be cloned
# due to lack of space.
cloneFailDbmName = tmpFilePat + "_CLONE_FAILED_DB"
rmFile(cloneFailDbmName + "*")
cloneFailDbm = ngamsDbm.ngamsDbm(cloneFailDbmName, cleanUpOnDestr=1,
writePerm=1)
# This is used to simulate disk completion. If a specific target
# disk is defined, all other disks should be ignored (=we set them to
# completed).
cloneSimDiskCompl = []
if (targetDiskId):
tmpDiskList = ngamsDiskUtils.\
getDiskInfoForMountedDisks(srvObj.getDb(), srvObj.getHostId(),
mtRootDir)
for idx in range(len(tmpDiskList)):
if (tmpDiskList[idx].getDiskId() != targetDiskId):
cloneSimDiskCompl.append(tmpDiskList[idx].getDiskId())
# Carry out the simulated clone process.
ngamsDiskUtils.findTargetDiskResetCache()
key = 0
while (1):
if (not cloneListDbm.hasKey(str(key))): break
fileInfo = cloneListDbm.get(str(key))
key += 1
fio = fileInfo[0]
hostName = fileInfo[1]
text = "Simulating cloning of file - File ID: %s/%d, on disk " +\
"with ID: %s on host: %s"
logger.debug(text, fio.getFileId(), fio.getFileVersion(),
fio.getDiskId(), hostName)
diskExemptList = cloneSimDiskCompl + [fio.getDiskId()]
trgDiskInfo = None
while (1):
try:
trgDiskInfo = ngamsDiskUtils.\
findTargetDisk(srvObj.getHostId(),
srvObj.getDb(), srvObj.getCfg(),
fio.getFormat(), 0,
diskExemptList, 1)
except Exception as e:
if (str(e).find("NGAMS_AL_NO_STO_SETS") != -1):
# No more candidate Target Disks for this type
# of data - this file cannot be cloned.
cloneFailDbm.addIncKey(fio)
break
# Check if a file with that ID + version is already
# stored on the selected Target Disk.
fileInDb = srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(),
fio.getFileId(),
fio.getFileVersion())
fileKey = ngamsLib.genFileKey(trgDiskInfo.getDiskId(),
fio.getFileId(),
fio.getFileVersion())
fileSimCloned = cloneSucDbm.hasKey(fileKey)
if (fileInDb or fileSimCloned):
# This file is already stored on the given disk.
# Add to the exempt list.
diskExemptList.append(trgDiskInfo.getDiskId())
else:
# OK, this disk should be OK, stop looking for a
# suitable Target Disk.
logger.debug("Found suitable disk with ID: %s/Slot ID: %s",
trgDiskInfo.getDiskId(), trgDiskInfo.getSlotId())
cloneSucDbm.add(fileKey, "")
break
# We now subtract the size of the file from the available amount of
# disk space for the selected Target Disk. When the amount of available
# space goes below the threshold defined for this NG/AMS system that
# disk is considered to be completed.
if (trgDiskInfo):
diskId = trgDiskInfo.getDiskId()
availDiskSpaceDic[diskId] -= float(fio.getFileSize())
if ((availDiskSpaceDic[diskId] / 1048576.0) < \
srvObj.getCfg().getFreeSpaceDiskChangeMb()):
cloneSimDiskCompl.append(diskId)
# Now, if there are files that could not be cloned we raise an exception
# indicating this, and in particular indicating for which mime-types there
# is not enough disk space.
if (cloneFailDbm.getCount()):
spaceLackMimeTypeDic = {}
cloneFailDbm.initKeyPtr()
while (1):
key, fio = cloneFailDbm.getNext()
if (not key): break
format = fio.getFormat()
if fio.getFormat() not in spaceLackMimeTypeDic:
spaceLackMimeTypeDic[format] = 0.0
spaceLackMimeTypeDic[format] += float(fio.getFileSize())
errMsg = "Insufficient space to carry out the CLONE Command. " +\
"Approximate amount of disk space missing (mime-type: MB):"
for mt in spaceLackMimeTypeDic.keys():
errMsg += " %s: %.3f MB" %\
(mt, (spaceLackMimeTypeDic[mt] / 1048576.0))
errMsg = genLog("NGAMS_ER_CLONE_REJECTED", [errMsg])
raise Exception(errMsg)
def archiveFromFile(srvObj,
filename,
noReplication=0,
mimeType=None,
reqPropsObj=None):
"""
Archive a file directly from a file as source.
srvObj: Reference to NG/AMS Server Object (ngamsServer).
filename: Name of file to archive (string).
noReplication: Flag to enable/disable replication (integer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
Returns: Execution result object of DAPI
"""
T = TRACE()
logger.debug("Archiving file: %s", filename)
logger.debug("Mimetype used is %s", mimeType)
if (reqPropsObj):
logger.debug("Request Properties Object given - using this")
reqPropsObjLoc = reqPropsObj
else:
logger.debug("No Request Properties Object given - creating one")
reqPropsObjLoc = ngamsArchiveUtils.ngamsReqProps.ngamsReqProps()
stagingFile = filename
try:
if (mimeType == None):
mimeType = ngamsHighLevelLib.determineMimeType(
srvObj.getCfg(), filename)
archive_start = time.time()
# Prepare dummy ngamsReqProps object (if an object was not given).
if (not reqPropsObj):
reqPropsObjLoc.setMimeType(mimeType)
reqPropsObjLoc.setStagingFilename(filename)
reqPropsObjLoc.setHttpMethod(NGAMS_HTTP_GET)
reqPropsObjLoc.setCmd(NGAMS_ARCHIVE_CMD)
reqPropsObjLoc.setSize(os.path.getsize(filename))
reqPropsObjLoc.setFileUri(NGAMS_HTTP_FILE_URL + filename)
reqPropsObjLoc.setNoReplication(noReplication)
# If no target disk is defined, find one suitable disk.
if (not reqPropsObjLoc.getTargDiskInfo()):
try:
trgDiskInfo = ngamsArchiveUtils.ngamsDiskUtils.\
findTargetDisk(srvObj.getHostId(),
srvObj.getDb(), srvObj.getCfg(),
mimeType, 0,
reqSpace=reqPropsObjLoc.getSize())
reqPropsObjLoc.setTargDiskInfo(trgDiskInfo)
# copy the file to the staging area of the target disk
stagingFile = trgDiskInfo.getMountPoint(
) + '/staging/' + os.path.basename(filename)
cpFile(filename, stagingFile)
reqPropsObjLoc.setStagingFilename(stagingFile)
except Exception as e:
errMsg = str(e) + ". Attempting to archive local file: " +\
filename
ngamsPlugInApi.notify(srvObj, NGAMS_NOTIF_NO_DISKS,
"NO DISKS AVAILABLE", errMsg)
raise Exception(errMsg)
# Set the log cache to 1 during the handling of the file.
plugIn = srvObj.getMimeTypeDic()[mimeType]
logger.debug("Invoking DAPI: %s to handle file: %s", plugIn,
stagingFile)
plugInMethod = loadPlugInEntryPoint(plugIn)
resMain = plugInMethod(srvObj, reqPropsObjLoc)
# Move the file to final destination.
st = time.time()
mvFile(reqPropsObjLoc.getStagingFilename(),
resMain.getCompleteFilename())
iorate = reqPropsObjLoc.getSize() / (time.time() - st)
ngamsArchiveUtils.postFileRecepHandling(srvObj, reqPropsObjLoc,
resMain, trgDiskInfo)
except Exception as e:
# If another error occurs, than one qualifying for Back-Log
# Buffering the file, we have to log an error.
if (ngamsHighLevelLib.performBackLogBuffering(srvObj.getCfg(),
reqPropsObjLoc, e)):
logger.exception(
"Tried to archive local file %s, keeping original file",
filename)
return [NGAMS_FAILURE, str(e), NGAMS_FAILURE]
else:
logger.exception("Tried to archive local file %s, " + \
"moving it to Bad Files Directory " + \
"-- cannot be handled", filename)
ngamsHighLevelLib.moveFile2BadDir(srvObj.getCfg(), filename)
# Remove pickle file if available.
pickleObjFile = filename + "." + NGAMS_PICKLE_FILE_EXT
if (os.path.exists(pickleObjFile)):
logger.debug("Removing Back-Log Buffer Pickle File: %s",
pickleObjFile)
rmFile(pickleObjFile)
return [NGAMS_FAILURE, str(e), NGAMS_FAILURE]
# If the file was handled successfully, we remove it from the
# Back-Log Buffer Directory unless the local file was a log-file
# in which case we leave the cleanup to the Janitor-Thread.
if stagingFile.find('LOG-ROTATE') > -1:
logger.debug("Successfully archived local file: %s", filename)
else:
logger.debug(
"Successfully archived local file: %s. Removing staging file.",
filename)
rmFile(stagingFile)
rmFile(stagingFile + "." + NGAMS_PICKLE_FILE_EXT)
logger.debug("Archived local file: %s. Time (s): %.3f", filename,
time.time() - archive_start)
return (resMain, trgDiskInfo, iorate)