def handleCmd(srvObj, reqPropsObj, httpRef):
"""
Handle REMDISK command. See also 'handleRemDisk()'.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
if (not srvObj.getCfg().getAllowRemoveReq()):
errMsg = genLog("NGAMS_ER_ILL_REQ", ["Remove"])
raise Exception(errMsg)
diskId = ""
execute = 0
if (reqPropsObj.hasHttpPar("disk_id")):
diskId = reqPropsObj.getHttpPar("disk_id")
if (diskId == ""):
errMsg = genLog("NGAMS_ER_CMD_SYNTAX",
[NGAMS_REMDISK_CMD, "Missing parameter: disk_id"])
raise Exception(errMsg)
if (not srvObj.getDb().diskInDb(diskId)):
errMsg = genLog("NGAMS_ER_UNKNOWN_DISK", [diskId])
raise Exception(errMsg)
if (reqPropsObj.hasHttpPar("execute")):
try:
execute = int(reqPropsObj.getHttpPar("execute"))
except:
errMsg = genLog("NGAMS_ER_REQ_HANDLING", ["Must provide proper " +\
"value for parameter: execute (0|1)"])
raise Exception(errMsg)
# Carry out the command.
status = remDisk(srvObj, reqPropsObj, diskId, execute)
# Send reply back to requestor (if not already done).
if (not status): return
xmlStat = status.genXmlDoc(0, 1, 1, 1, 0)
xmlStat = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, xmlStat)
if (status.getStatus() == NGAMS_SUCCESS):
httpStat = NGAMS_HTTP_SUCCESS
else:
httpStat = NGAMS_HTTP_BAD_REQ
httpRef.send_data(six.b(xmlStat), NGAMS_XML_MT, code=httpStat)
def handleCmd(srvObj, reqPropsObj, httpRef):
"""
Handle REMFILE command. See also 'handleRemFile()'.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
T = TRACE()
if (not srvObj.getCfg().getAllowRemoveReq()):
errMsg = genLog("NGAMS_ER_ILL_REQ", ["Remove"])
raise Exception(errMsg)
diskId = ""
fileId = ""
fileVersion = -1
execute = 0
if (reqPropsObj.hasHttpPar("disk_id")):
diskId = reqPropsObj.getHttpPar("disk_id")
if (reqPropsObj.hasHttpPar("file_id")):
fileId = reqPropsObj.getHttpPar("file_id")
if (reqPropsObj.hasHttpPar("file_version")):
fileVersion = int(reqPropsObj.getHttpPar("file_version"))
if (reqPropsObj.hasHttpPar("execute")):
try:
execute = int(reqPropsObj.getHttpPar("execute"))
except:
errMsg = genLog("NGAMS_ER_REQ_HANDLING", ["Must provide proper " +\
"value for parameter: execute (0|1)"])
raise Exception(errMsg)
# Carry out the command.
status = remFile(srvObj, reqPropsObj, diskId, fileId, fileVersion, execute)
# Send reply back to requestor.
xmlStat = status.genXmlDoc(0, 1, 1, 1, 0)
xmlStat = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, xmlStat)
httpRef.send_data(xmlStat, NGAMS_XML_MT)
def handleCmd(srvObj,
reqPropsObj,
httpRef):
"""
Handle STATUS command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler object
(ngamsHttpRequestHandler).
Returns: Void.
"""
status = ngamsStatus.ngamsStatus()
status.setDate(toiso8601()).\
setVersion(getNgamsVersion()).setHostId(srvObj.getHostId()).\
setStatus(NGAMS_SUCCESS).\
setMessage("Successfully handled command STATUS").\
setState(srvObj.getState()).setSubState(srvObj.getSubState())
reqPropsObjRef = reqPropsObj
# Get the information requested.
diskId = ""
fileId = ""
fileVersion = ""
configurationFile = ""
fileAccess = ""
hostId = ""
requestId = ""
dbTime = ""
dbTimeReset = ""
fileList = ""
fileListId = ""
maxElements = 100000
if (reqPropsObj.hasHttpPar("disk_id")):
diskId = reqPropsObj.getHttpPar("disk_id")
if (reqPropsObj.hasHttpPar("file_id")):
fileId = reqPropsObj.getHttpPar("file_id")
if (reqPropsObj.hasHttpPar("file_version")):
fileVersion = int(reqPropsObj.getHttpPar("file_version"))
if (reqPropsObj.hasHttpPar("configuration_file")):
configurationFile = reqPropsObj.getHttpPar("configuration_file")
if (reqPropsObj.hasHttpPar("file_access")):
fileAccess = reqPropsObj.getHttpPar("file_access")
if (reqPropsObj.hasHttpPar("host_id")):
hostId = reqPropsObj.getHttpPar("host_id")
if (reqPropsObj.hasHttpPar("max_elements")):
maxElements = int(reqPropsObj.getHttpPar("max_elements"))
if (reqPropsObj.hasHttpPar("request_id")):
requestId = reqPropsObj.getHttpPar("request_id")
if (reqPropsObj.hasHttpPar("db_time")):
dbTime = True
if (reqPropsObj.hasHttpPar("db_time_reset")):
dbTimeReset = True
if (reqPropsObj.hasHttpPar("flush_log")):
# in the past this called flushLog()
# do we really want to keep that functionality? I doubt it
pass
if (reqPropsObj.hasHttpPar("file_list")):
fileList = int(reqPropsObj.getHttpPar("file_list"))
if (reqPropsObj.hasHttpPar("file_list_id")):
fileListId = reqPropsObj.getHttpPar("file_list_id")
if (reqPropsObj.hasHttpPar("dump_object_info")):
# Dump status of all objects allocated into file specified.
targFile = reqPropsObj.getHttpPar("dump_object_info")
rmFile(targFile)
fo = open(targFile, "w")
fo.write(_genObjectStatus())
fo.close()
# Handle request for file info.
genCfgStatus = 0
genDiskStatus = 0
genFileStatus = 0
genStatesStatus = 1
genRequestStatus = 0
msg = ""
help = 0
if (reqPropsObj.hasHttpPar("help")):
global _help
msg = _help
help = 1
elif hostId and hostId != srvObj.getHostId():
host, port = srvObj.get_remote_server_endpoint(hostId)
cfgObj = srvObj.getCfg()
if not cfgObj.getProxyMode():
httpRef.redirect(host, port)
return
else:
try:
httpRef.proxy_request(hostId, host, port)
except Exception as e:
ex = re.sub("<|>", "", str(e))
errMsg = genLog("NGAMS_ER_COM",
[host, port,ex])
raise Exception(errMsg)
return
elif (fileList):
if (not fileListId):
# It's a new STATUS?file_list request.
fileListId = _handleFileList(srvObj, reqPropsObj, httpRef)
# Send back data from the request.
_handleFileListReply(srvObj, reqPropsObj, httpRef, fileListId,
maxElements)
elif (diskId):
diskObj = ngamsDiskInfo.ngamsDiskInfo()
diskObj.read(srvObj.getDb(), diskId)
status.addDiskStatus(diskObj)
genDiskStatus = 1
elif (fileId):
if (not fileVersion): fileVersion = -1
try:
fileObj = ngamsFileInfo.ngamsFileInfo()
fileObj.read(srvObj.getHostId(), srvObj.getDb(), fileId, fileVersion)
diskObj = ngamsDiskInfo.ngamsDiskInfo()
try:
diskObj.read(srvObj.getDb(), fileObj.getDiskId())
except:
errMsg = "Illegal Disk ID found: {0} for file with ID: {1}".format(fileObj.getDiskId(), fileId)
raise Exception(errMsg)
diskObj.addFileObj(fileObj)
status.addDiskStatus(diskObj)
except:
# The file was not found in the database. Check if it is available
# on a remote partner site.
host, port, status_info, disk_info, file_info = \
ngamsFileUtils.lookup_partner_site_file_status(srvObj, fileId,
fileVersion,
reqPropsObj)
# Update status reply using the disk status and file status
# information retrieved from the partner site
status.addDiskStatus(disk_info)
genDiskStatus = 1
genFileStatus = 1
elif (requestId):
logger.debug("Checking status of request with ID: %s", requestId)
reqPropsObjRef = srvObj.getRequest(requestId)
if (not reqPropsObjRef):
errMsg = genLog("NGAMS_ER_ILL_REQ_ID", [requestId])
raise Exception(errMsg)
genRequestStatus = 1
elif (configurationFile):
msg = "configuration_file=" + srvObj.cfg_fname
genCfgStatus = 1
# Hidden feature to return complete Cfg!!
try:
genCfgStatus = int(configurationFile)
except ValueError:
genCfgStatus = 1
status.setNgamsCfgObj(srvObj.getCfg())
elif (fileAccess):
if (not fileVersion): fileVersion = -1
fileId = fileAccess
msg = _checkFileAccess(srvObj, reqPropsObj, httpRef, fileId,
fileVersion, diskId)
elif (dbTime):
logger.debug("Querying total DB time")
msg = "Total DB time: %.6fs" % srvObj.getDb().getDbTime()
elif (dbTimeReset):
msg = "Resetting DB timer"
logger.debug(msg)
srvObj.getDb().resetDbTime()
else:
msg = "Successfully handled command STATUS"
if (reqPropsObjRef == reqPropsObj):
reqPropsObj.setCompletionTime()
srvObj.updateRequestDb(reqPropsObj)
if (genCfgStatus or genDiskStatus or genFileStatus or genRequestStatus):
status.setReqStatFromReqPropsObj(reqPropsObjRef)
# Generate XML reply.
logger.debug("Status requests: " + str([genCfgStatus, genDiskStatus, genFileStatus,
genStatesStatus]))
xmlStat = status.genXmlDoc(genCfgStatus, genDiskStatus, genFileStatus,
genStatesStatus)
xmlStat = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, xmlStat)
httpRef.send_data(six.b(xmlStat), NGAMS_XML_MT)
elif not httpRef.reply_sent:
httpRef.send_status(msg)
if (msg and (not help)):
logger.info(msg)
else:
logger.info("Successfully handled command STATUS")
def _clone(srvObj, diskId, fileId, fileVersion, targetDiskId, reqPropsObj,
httpRef, tmpFilePat):
"""
Internal function used by ngamsCloneCmd.clone() to carry out the
cloning. See documentation for ngamsCloneCmd.clone().
"""
targetDiskId = targetDiskId.strip()
logger.debug("Handling file cloning with parameters - File ID: %s -" + \
"Disk ID: %s - File Version: %s - Target Disk ID: |%s|",
fileId, diskId, str(fileVersion), targetDiskId)
if (((fileId == "") and (diskId == "") and (fileVersion != -1))
or ((fileId == "") and (diskId == "") and (fileVersion == -1))):
errMsg = genLog("NGAMS_ER_CMD_SYNTAX",
[NGAMS_CLONE_CMD, "File Id: " + fileId +\
", Disk ID: " + diskId +\
", File Version: " + str(fileVersion)])
raise Exception, errMsg
# If Disk ID, File ID and File Version are given, execute a quick cloning.
try:
fileVersion = int(fileVersion)
except:
pass
if (False and diskId and fileId and (fileVersion > 0)):
_cloneExplicit(srvObj, reqPropsObj, diskId, fileId, fileVersion,
targetDiskId)
logger.info("Successfully handled command CLONE")
return
# Handling cloning of more files.
cloneListDbm = None
cloneListDbmName = tmpFilePat + "_CLONE_INFO_DB"
try:
# Get information about candidate files for cloning.
files = srvObj.db.getFileInfoFromFileId(fileId,
fileVersion,
diskId,
ignore=0,
order=0,
dbCursor=False)
if not files:
msg = genLog("NGAMS_ER_CMD_EXEC",
[NGAMS_CLONE_CMD, "No files for cloning found"])
raise Exception(msg)
# Convert to tuple of file info object plus extra info
# This is how the code expects this information to come, so we need to
# keep the format unless we change the bulk of the code
# f[-2] is the host id, f[-1] is the mount point
all_info = []
for f in files:
all_info.append((ngamsFileInfo.ngamsFileInfo().unpackSqlResult(f),
f[-2], f[-1]))
# Create a BSD DB with information about files to be cloned.
rmFile(cloneListDbmName + "*")
cloneListDbm = ngamsDbm.ngamsDbm(cloneListDbmName,
cleanUpOnDestr=0,
writePerm=1)
cloneDbCount = 0
if fileId != "" and (diskId != "" or fileVersion == -1):
# Take only the first element
cloneListDbm.add("0", all_info[0])
cloneDbCount = 1
else:
# Take all the files.
for tmpFileInfo in all_info:
cloneListDbm.add(str(cloneDbCount), tmpFileInfo)
cloneDbCount += 1
except Exception:
if (cloneListDbm): del cloneListDbm
rmFile(cloneListDbmName + "*")
raise
logger.debug("Found: %d file(s) for cloning ...", cloneDbCount)
del cloneListDbm
# Check available amount of disk space.
cloneCheckDiskSpace(srvObj, cloneListDbmName, tmpFilePat, targetDiskId)
# Initialize Request Status parameters.
if (reqPropsObj):
reqPropsObj.\
setCompletionPercent(0, 1).\
setExpectedCount(cloneDbCount, 1).\
setActualCount(0, 1)
srvObj.updateRequestDb(reqPropsObj)
# Wait until CLONE Command has finished, or send a reply before cloning?
async = 'async' in reqPropsObj and int(reqPropsObj['async'])
if async:
# Send intermediate reply if the HTTP Reference object is given
# whenever send an auto reply now.
logger.debug("CLONE command accepted - generating immediate " +
"confimation reply to CLONE command")
status = srvObj.genStatus(NGAMS_SUCCESS,
"Accepted CLONE command for execution").\
setReqStatFromReqPropsObj(reqPropsObj).\
setActualCount(0)
# Do the actual cloning in a thread
args = (srvObj, cloneListDbmName, tmpFilePat, targetDiskId,
reqPropsObj, None)
thrName = NGAMS_CLONE_THR + threading.current_thread().getName()
cloneThread = threading.Thread(None, _cloneThread, thrName, args)
cloneThread.setDaemon(0)
cloneThread.start()
else:
# Carry out the cloning (directly in this thread) and send reply
# when this is done.
_cloneExec(srvObj, cloneListDbmName, tmpFilePat, targetDiskId,
reqPropsObj)
msg = "Successfully handled command CLONE"
logger.debug(msg)
status = srvObj.genStatus(NGAMS_SUCCESS, msg).\
setReqStatFromReqPropsObj(reqPropsObj).setActualCount(0)
rmFile(cloneListDbmName + "*")
# Send reply if possible.
if (httpRef):
xmlStat = status.genXmlDoc(0, 0, 0, 1, 0)
xmlStat = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, xmlStat)
httpRef.send_data(xmlStat, NGAMS_XML_MT)
cloneStatusFileList.addFileListObj(tmpFileList)
# Make overall status.
cloneStatusFileList.setStatus("SUCCESS: " +\
str(successCloneCount) +\
", FAILURE: " +\
str(failedCloneCount) +\
", NOT DONE: " +\
str(len(cloneStatusFileList) -\
successCloneCount -\
failedCloneCount))
status = srvObj.genStatus(NGAMS_SUCCESS,
"CLONE command status report").\
addFileList(cloneStatusFileList)
statRep = status.genXmlDoc(0, 0, 0, 1, 0)
statRep = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, statRep)
mimeType = NGAMS_XML_MT
else:
# Generate a 'simple' ASCII report.
statRep = tmpFilePat + "_NOTIF_EMAIL.txt"
fo = open(statRep, "w")
if (reqPropsObj.hasHttpPar("disk_id")):
diskId = reqPropsObj.getHttpPar("disk_id")
else:
diskId = "-----"
if (reqPropsObj.hasHttpPar("file_id")):
fileId = reqPropsObj.getHttpPar("file_id")
else:
fileId = "-----"
if (reqPropsObj.hasHttpPar("file_version")):
fileVersion = reqPropsObj.getHttpPar("file_version")
def _clone(srvObj,
diskId,
fileId,
fileVersion,
targetDiskId,
reqPropsObj,
httpRef,
tmpFilePat):
"""
Internal function used by ngamsCloneCmd.clone() to carry out the
cloning. See documentation for ngamsCloneCmd.clone().
"""
targetDiskId = targetDiskId.strip()
logger.debug("Handling file cloning with parameters - File ID: %s -" + \
"Disk ID: %s - File Version: %s - Target Disk ID: |%s|",
fileId, diskId, str(fileVersion), targetDiskId)
if not fileId and not diskId:
errMsg = genLog("NGAMS_ER_CMD_SYNTAX",
[NGAMS_CLONE_CMD, "File Id: " + fileId +\
", Disk ID: " + diskId +\
", File Version: " + str(fileVersion)])
raise Exception(errMsg)
# Get candidate files for cloning.
# We convert to tuple of file info object plus extra info because
# this is how the code expects this information to come, so we need to
# keep the format unless we change the bulk of the code
# f[-2] is the host id, f[-1] is the mount point
all_info = [(ngamsFileInfo.ngamsFileInfo().unpackSqlResult(f), f[-2], f[-1])
for f in srvObj.db.getFileInfoFromFileId(
fileId, fileVersion, diskId, ignore=0, order=0, order2=1,
dbCursor=False)]
if not all_info:
msg = genLog("NGAMS_ER_CMD_EXEC",
[NGAMS_CLONE_CMD, "No files for cloning found"])
raise Exception(msg)
# Take only the first element in these cases
if fileId != "" and (diskId != "" or fileVersion == -1):
all_info = [all_info[0]]
cloneDbCount = len(all_info)
cloneListDbmName = tmpFilePat + "_CLONE_INFO_DB"
rmFile(cloneListDbmName + "*")
try:
ngamsDbm.enumerate_to_dbm(cloneListDbmName, all_info)
except Exception:
rmFile(cloneListDbmName + "*")
raise
logger.debug("Found: %d file(s) for cloning ...", cloneDbCount)
# Check available amount of disk space.
cloneCheckDiskSpace(srvObj, cloneListDbmName, tmpFilePat, targetDiskId)
# Initialize Request Status parameters.
if (reqPropsObj):
reqPropsObj.\
setCompletionPercent(0, 1).\
setExpectedCount(cloneDbCount, 1).\
setActualCount(0, 1)
srvObj.updateRequestDb(reqPropsObj)
# Wait until CLONE Command has finished, or send a reply before cloning?
is_async = 'async' in reqPropsObj and int(reqPropsObj['async'])
if is_async:
# Send intermediate reply if the HTTP Reference object is given
# whenever send an auto reply now.
logger.debug("CLONE command accepted - generating immediate " +
"confimation reply to CLONE command")
status = srvObj.genStatus(NGAMS_SUCCESS,
"Accepted CLONE command for execution").\
setReqStatFromReqPropsObj(reqPropsObj).\
setActualCount(0)
# Do the actual cloning in a thread
args = (srvObj, cloneListDbmName, tmpFilePat, targetDiskId,
reqPropsObj, None)
thrName = NGAMS_CLONE_THR + threading.current_thread().getName()
cloneThread = threading.Thread(None, _cloneThread, thrName, args)
cloneThread.daemon = False
cloneThread.start()
else:
# Carry out the cloning (directly in this thread) and send reply
# when this is done.
_cloneExec(srvObj, cloneListDbmName, tmpFilePat, targetDiskId,
reqPropsObj)
msg = "Successfully handled command CLONE"
logger.debug(msg)
status = srvObj.genStatus(NGAMS_SUCCESS, msg).\
setReqStatFromReqPropsObj(reqPropsObj).setActualCount(0)
rmFile(cloneListDbmName + "*")
# Send reply if possible.
if (httpRef):
xmlStat = status.genXmlDoc(0, 0, 0, 1, 0)
xmlStat = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, xmlStat)
httpRef.send_data(six.b(xmlStat), NGAMS_XML_MT)
def _cloneExec(srvObj,
cloneListDbmName,
tmpFilePat,
targetDiskId,
reqPropsObj):
"""
See documentation of ngamsCloneCmd._cloneThread(). This function is
merely implemented in order to encapsulate the whole process to be able
to clean up properly when the processing is terminated.
"""
cloneStatusDbm = None
emailNotif = 0
checkChecksum = 1
if (reqPropsObj):
if (reqPropsObj.hasHttpPar("notif_email")): emailNotif = 1
if (reqPropsObj.hasHttpPar("check")):
checkChecksum = int(reqPropsObj.getHttpPar("check"))
# Open clone list DB.
cloneListDbm = ngamsDbm.ngamsDbm(cloneListDbmName)
# We have to get the port numbers of the hosts where the files to be
# cloned are stored.
hostInfoDic = {}
cloneListDbm.initKeyPtr()
while (1):
key, fileInfo = cloneListDbm.getNext()
if (not key): break
hostInfoDic[fileInfo[1]] = -1
hostInfoDic = ngamsHighLevelLib.resolveHostAddress(srvObj.getHostId(),
srvObj.getDb(),
srvObj.getCfg(),
hostInfoDic.keys())
# The cloning loop. Loop over the list of files to clone and generate
# a report with the result.
if (emailNotif):
cloneStatusDbmName = tmpFilePat + "_CLONE_STATUS_DB"
cloneStatusDbm = ngamsDbm.ngamsDbm(cloneStatusDbmName,
cleanUpOnDestr = 0, writePerm = 1)
successCloneCount = 0
failedCloneCount = 0
abortCloneLoop = 0
timeAccu = 0.0
key = 0
while (1):
clone_start = time.time()
if (not cloneListDbm.hasKey(str(key))): break
fileInfo = cloneListDbm.get(str(key))
key += 1
# Check if we have permission to run. Otherwise, stop.
if (not srvObj.run_async_commands): break
fio = fileInfo[0]
mtPt = fileInfo[2]
if (emailNotif):
tmpFileList = ngamsFileList.\
ngamsFileList("FILE_CLONE_STATUS",
"File: " + fio.getFileId() + "/" +\
fio.getDiskId() + "/" +\
str(fio.getFileVersion()))
hostId = fileInfo[1]
text = "Cloning file - File ID: %s/%d, on disk " +\
"with ID: %s on host: %s"
logger.debug(text, fio.getFileId(), fio.getFileVersion(),
fio.getDiskId(), hostId)
# We generate a local Staging File and archive this.
stagingFilename = ""
try:
# Check if file is marked as bad.
if (fio.getFileStatus()[0] == "1"):
errMsg = "File marked as bad - skipping!"
raise Exception(errMsg)
if (targetDiskId == ""):
# Try to find a disk not hosting already a file with that
# ID + version.
diskExemptList = [fio.getDiskId()]
while (1):
trgDiskInfo = ngamsDiskUtils.\
findTargetDisk(srvObj.getHostId(),
srvObj.getDb(),
srvObj.getCfg(),
fio.getFormat(),
1, diskExemptList)
# Check if a file with that ID + version is already
# stored on the selected Target Disk.
if (srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(),
fio.getFileId(),
fio.getFileVersion())):
# This file is already stored on the given disk.
# Add to the exempt list.
diskExemptList.append(trgDiskInfo.getDiskId())
else:
# OK, this disk should be OK, stop looking for a
# suitable Target Disk.
break
else:
try:
trgDiskInfo = ngamsDiskInfo.ngamsDiskInfo().\
read(srvObj.getDb(), targetDiskId)
slotId = trgDiskInfo.getSlotId()
storageSetId = srvObj.getCfg().\
getStorageSetFromSlotId(slotId).\
getStorageSetId()
trgDiskInfo.setStorageSetId(storageSetId)
except Exception:
abortCloneLoop = 1
raise
# We don't accept to clone onto the same disk (this would mean
# overwriting).
if (trgDiskInfo.getDiskId() == fio.getDiskId()):
err = "Source and target files are identical"
msg = "Failed in cloning file with ID: " + fio.getFileId() +\
"/Version: " + str(fio.getFileVersion()) +\
" on disk with ID: " + fio.getDiskId() +\
" on host: " + hostId + ". Reason: " + err
logger.warning(msg)
if (emailNotif):
tmpFileList.setStatus(NGAMS_FAILURE + ": " + err)
tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE"))
cloneStatusDbm.addIncKey(tmpFileList)
failedCloneCount += 1
continue
tmpReqPropsObj = ngamsReqProps.ngamsReqProps()
tmpReqPropsObj.setMimeType(fio.getFormat())
stagingFilename = ngamsHighLevelLib.\
genStagingFilename(srvObj.getCfg(),
tmpReqPropsObj,
trgDiskInfo, fio.getFileId())
# Receive the data into the Staging File using the urllib.
if (srvObj.getHostId() != hostId):
# Example: http://host:7777/RETRIEVE?file_id=id&file_version=1
ipAddress = hostInfoDic[hostId].getIpAddress()
portNo = hostInfoDic[hostId].getSrvPort()
fileUrl = "http://" + ipAddress + ":" + str(portNo) +\
"/RETRIEVE?" + "file_id=" + fio.getFileId() +\
"&file_version=" + str(fio.getFileVersion())
# If a specific Disk ID for the source file is given, append
# this.
if (fio.getDiskId()):
fileUrl += "&disk_id=%s" % fio.getDiskId()
# Check if host is suspended, if yes, wake it up.
if (srvObj.getDb().getSrvSuspended(hostId)):
logger.debug("Clone Request - Waking up suspended " +\
"NGAS Host: %s", hostId)
ngamsSrvUtils.wakeUpHost(srvObj, hostId)
else:
fileUrl = "file:" + mtPt + "/" + fio.getFilename()
logger.debug("Receiving file via URI: %s into staging filename: %s",
fileUrl, stagingFilename)
# We try up to 5 times to retrieve the file in case a problem is
# encountered during cloning.
for attempt in range(5):
try:
filename, headers = urlrequest.urlretrieve(fileUrl, stagingFilename)
_checkFile(srvObj, fio, stagingFilename, headers,
checkChecksum)
# If we get to this point the transfer was (probably) OK.
break
except Exception as e:
rmFile(stagingFilename)
errMsg = "Problem occurred while cloning file "+\
"via URL: " + fileUrl + " - Error: " + str(e)
if (attempt < 4):
errMsg += " - Retrying in 5s ..."
logger.error(errMsg)
time.sleep(5)
else:
raise Exception(errMsg)
# We simply copy the file into the same destination as the
# source file (but on another disk).
targPathName = os.path.dirname(fio.getFilename())
targFilename = os.path.basename(fio.getFilename())
complTargPath = os.path.normpath(trgDiskInfo.getMountPoint() +\
"/" + targPathName)
checkCreatePath(complTargPath)
complFilename = os.path.normpath(complTargPath + "/"+targFilename)
mvTime = mvFile(stagingFilename, complFilename)
ngamsLib.makeFileReadOnly(complFilename)
# Update status for new file in the DB.
newFileInfo = fio.clone().setDiskId(trgDiskInfo.getDiskId()).\
setCreationDate(getFileCreationTime(complFilename))
fileExists = srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(),
fio.getFileId(),
fio.getFileVersion())
newFileInfo.write(srvObj.getHostId(), srvObj.getDb())
# Update status for the Target Disk in DB + check if the disk is
# completed.
if (fileExists): mvTime = 0
dummyDapiStatObj = ngamsDapiStatus.ngamsDapiStatus().\
setDiskId(trgDiskInfo.getDiskId()).\
setFileExists(fileExists).\
setFileSize(fio.getFileSize()).setIoTime(mvTime)
ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), dummyDapiStatObj)
ngamsArchiveUtils.checkDiskSpace(srvObj, trgDiskInfo.getDiskId())
# Update the clone file status list.
if (emailNotif):
tmpFileList.setStatus(NGAMS_SUCCESS)
tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE"))
tmpFileList.addFileInfoObj(newFileInfo.setTag("TARGET_FILE"))
cloneStatusDbm.addIncKey(tmpFileList)
successCloneCount += 1
# If running as a cache archive, update the Cache New Files DBM
# with the information about the new file.
if (srvObj.getCachingActive()):
diskId = trgDiskInfo.getDiskId()
fileId = fio.getFileId()
fileVer = fio.getFileVersion()
filename = fio.getFilename()
ngamsCacheControlThread.addEntryNewFilesDbm(srvObj, diskId,
fileId, fileVer,
filename)
# Generate a confirmation log entry.
cloneTime = time.time() - clone_start
timeAccu += cloneTime
msg = genLog("NGAMS_INFO_FILE_CLONED",
[fio.getFileId(), fio.getFileVersion(),
fio.getDiskId(), hostId])
msg = msg + ". Time: %.3fs. Total time: %.3fs." %\
(cloneTime, timeAccu)
logger.info(msg, extra={'to_syslog': True})
except Exception as e:
cloneTime = time.time() - clone_start
timeAccu += cloneTime
errMsg = genLog("NGAMS_ER_FILE_CLONE_FAILED",
[fio.getFileId(), fio.getFileVersion(),
fio.getDiskId(), hostId, str(e)])
if (abortCloneLoop):
logger.error(errMsg, extra={'to_syslog': True})
return
else:
logger.warning(errMsg)
if (emailNotif):
tmpFileList.setStatus(NGAMS_FAILURE + ": Error: " + errMsg)
tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE"))
cloneStatusDbm.addIncKey(tmpFileList)
failedCloneCount += 1
# Delete Staging File if already created.
if ((stagingFilename != "") and (os.path.exists(stagingFilename))):
rmFile(stagingFilename)
# Calculate time statistics.
if (reqPropsObj):
ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj.\
incActualCount(1), timeAccu)
# Final update of the Request Status.
if (reqPropsObj):
complPercent = (100.0 * (float(reqPropsObj.getActualCount()) /
float(reqPropsObj.getExpectedCount())))
reqPropsObj.setCompletionPercent(complPercent, 1)
reqPropsObj.setCompletionTime(1)
srvObj.updateRequestDb(reqPropsObj)
# Send Clone Report with list of files cloned to a possible
# requestor(select) of this.
totFiles = (successCloneCount + failedCloneCount)
if (emailNotif):
xmlStat = 0
# TODO: Generation of XML status report is disabled since we cannot
# handle for the moment XML documents with 1000s of elements.
if (xmlStat):
cloneStatusFileList = ngamsFileList.\
ngamsFileList("FILE_CLONING_STATUS_REPORT",
"File Cloning Status Report")
fileCount = 0
while (fileCount < cloneStatusDbm.getCount()):
tmpFileList = cloneStatusDbm.get(str(fileCount))
cloneStatusFileList.addFileListObj(tmpFileList)
# Make overall status.
cloneStatusFileList.setStatus("SUCCESS: " +\
str(successCloneCount) +\
", FAILURE: " +\
str(failedCloneCount) +\
", NOT DONE: " +\
str(len(cloneStatusFileList) -\
successCloneCount -\
failedCloneCount))
status = srvObj.genStatus(NGAMS_SUCCESS,
"CLONE command status report").\
addFileList(cloneStatusFileList)
statRep = status.genXmlDoc(0, 0, 0, 1, 0)
statRep = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, statRep)
mimeType = NGAMS_XML_MT
else:
# Generate a 'simple' ASCII report.
statRep = tmpFilePat + "_NOTIF_EMAIL.txt"
fo = open(statRep, "w")
if (reqPropsObj.hasHttpPar("disk_id")):
diskId = reqPropsObj.getHttpPar("disk_id")
else:
diskId = "-----"
if (reqPropsObj.hasHttpPar("file_id")):
fileId = reqPropsObj.getHttpPar("file_id")
else:
fileId = "-----"
if (reqPropsObj.hasHttpPar("file_version")):
fileVersion = reqPropsObj.getHttpPar("file_version")
else:
fileVersion = "-----"
tmpFormat = "CLONE STATUS REPORT:\n\n" +\
"==Summary:\n\n" +\
"Date: %s\n" +\
"NGAS Host: %s\n" +\
"Disk ID: %s\n" +\
"File ID: %s\n" +\
"File Version: %s\n" +\
"Total Number of Files: %d\n" +\
"Number of Cloned Files: %d\n" +\
"Number of Failed Files: %d\n" +\
"Total processing time (s): %.3f\n" +\
"Handling time per file (s): %.3f\n\n" +\
"==File List:\n\n"
fo.write(tmpFormat % (toiso8601(), srvObj.getHostId(), diskId, fileId,
str(fileVersion), totFiles,
successCloneCount, failedCloneCount,
timeAccu, (timeAccu / totFiles)))
tmpFormat = "%-70s %-70s %-7s\n"
fo.write(tmpFormat % ("Source File", "Target File", "Status"))
fo.write(tmpFormat % (70 * "-", 70 * "-", 7 * "-"))
key = 1
while (1):
if (not cloneStatusDbm.hasKey(str(key))): break
tmpFileList = cloneStatusDbm.get(str(key))
key += 1
srcFileObj = tmpFileList.getFileInfoObjList()[0]
srcFile = "%s/%s/%d" % (srcFileObj.getDiskId(),
srcFileObj.getFileId(),
srcFileObj.getFileVersion())
if (tmpFileList.getStatus() == NGAMS_SUCCESS):
trgFileObj = tmpFileList.getFileInfoObjList()[1]
trgFile = "%s/%s/%d" % (trgFileObj.getDiskId(),
trgFileObj.getFileId(),
trgFileObj.getFileVersion())
else:
trgFile = "-----"
fo.write(tmpFormat % (srcFile,trgFile,tmpFileList.getStatus()))
fo.write(149 * "-")
fo.write("\n\n==END\n")
fo.close()
mimeType = NGAMS_TEXT_MT
# Send out the status report.
emailAdrList = reqPropsObj.getHttpPar("notif_email").split(",")
attachmentName = "CloneStatusReport"
if (reqPropsObj.hasHttpPar("disk_id")):
attachmentName += "-" + reqPropsObj.getHttpPar("disk_id")
if (reqPropsObj.hasHttpPar("file_id")):
attachmentName += "-" + reqPropsObj.getHttpPar("file_id")
if (reqPropsObj.hasHttpPar("file_version")):
attachmentName += "-" + reqPropsObj.getHttpPar("file_version")
ngamsNotification.notify(srvObj.host_id, srvObj.cfg, NGAMS_NOTIF_INFO,
"CLONE STATUS REPORT", statRep, recList=emailAdrList, force=1,
contentType=mimeType, attachmentName=attachmentName)
del cloneStatusDbm
rmFile(cloneStatusDbmName + "*")
rmFile(statRep)
if (cloneListDbm): del cloneListDbm
rmFile(cloneListDbmName + "*")
logger.info("_cloneExec(). Total time: %.3fs. Average time per file: %.3fs.",
timeAccu, (timeAccu / totFiles))
def _registerExec(srvObj,
fileListDbmName,
tmpFilePat,
diskInfoDic,
reqPropsObj=None):
"""
Register the files listed in the File List DBM (ngamsDbm), which
match the mime-type(s) either specified in the 'mimeType' parameter,
or if this is not specified, which match all the mime-types specified
in the configuration file.
When the registration procedure has been executed, the function sends
an Email Notification message indicating which files were registered
if the HTTP parameter 'notif_email' is given.
The functions creates a File Info Objects per file handled and
writes this in a temporary file, which is a DBM file. The
keys in this DB is simply the file number in the sequence of files
handled, pointing to a pickled ngamsFileInfo object.
Each of the File Info Objects indicates if the file was registered or not.
This is done by setting the tag of the File Info Object to one of the
following values:
REGISTERED: The file was successfully registered in the NGAS DB
FAILED: The file was selected for registration but could not
be properly registered because of inconsistencies.
The status will be of the format: 'FAILED[: <reason>]'.
REJECTED: A file found under the specified path directory was
not accepted for cloning, usually because the mime-type
was not correct. The status will be of the format:
'REJECTED[: <reason>]'.
Note, that registration is comparable to archiving of files. For that
reason a DAPI must be provided for each type of file that should be
registered. If this is not fullfilled, the file registration will
fail.
Only files stored on one of the NGAS disks configured in the
configuration file, are considered. Files stored in other locations
are ignored.
srvObj: Instance of NG/AMS Server object (ngamsServer).
fileListDbmName: Name of a DBM containing the information
about the files to be registered. Each element in the
list is referred to by a key, which is a number.
These points to a pickled list for each file containing
the following information:
[<Filename>, <Disk ID>, <Mime-Type>]
The information for each disk concerned is also contained
in the DB referred to by its Disk ID and by its mount
point. The data is a pickled instance of the ngamsDiskInfo
class (string).
tmpFilePat: Pattern for temporary files used during the registration
process (string).
diskInfoDic: Dictionary with Disk IDs as keys pointing to the info
about the disk (dictionary/ngamsDiskInfo).
reqPropsObj: If an NG/AMS Request Properties Object is given, the
Request Status will be updated as the request is carried
out (ngamsReqProps).
Returns: Void.
"""
emailNotif = 0
if (reqPropsObj):
if (reqPropsObj.hasHttpPar("notif_email")):
emailNotif = 1
# Create the temporary BSD DB to contain the information for the
# Email Notification Message.
if (emailNotif):
regDbmName = tmpFilePat + "_NOTIF_EMAIL"
regDbm = ngamsDbm.ngamsDbm(regDbmName, writePerm=1)
# Open the DBM containing the list of files to (possibly) register.
fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm=1)
# Want to parse files in alphabetical order.
# TODO: Portatibility issue. Try to avoid UNIX shell commands for sorting.
tmpFileList = tmpFilePat + "_FILE_LIST"
rmFile(tmpFileList)
with open(tmpFileList, "wb") as fo:
fileListDbm.initKeyPtr()
while (1):
dbmKey, fileInfo = fileListDbm.getNext()
if (not dbmKey): break
fo.write(dbmKey + b"\n")
sortFileList = tmpFilePat + "_SORT_FILE_LIST"
rmFile(sortFileList)
shellCmd = "sort %s > %s" % (tmpFileList, sortFileList)
stat, out, err = ngamsCore.execCmd(shellCmd)
if (stat != 0):
raise Exception("Error executing command: %s. Error: %s, %s" %\
(shellCmd, str(out), str(err)))
rmFile(tmpFileList)
# Go through each file in the list, check if the mime-type is among the
# ones, which apply for registration. If yes try to register the file
# by invoking the corresponding DAPI on the file.
fileRegCount = 0
fileFailCount = 0
fileRejectCount = 0
regTimeAccu = 0.0
fileCount = 0
fo = open(sortFileList)
run = 1
while (run):
reg_start = time.time()
dbmKey = fo.readline()
if (dbmKey.strip() == ""):
run = 0
continue
fileInfo = fileListDbm.get(dbmKey[0:-1])
filename = fileInfo[0]
diskId = fileInfo[1]
mimeType = fileInfo[2]
# Register the file. Check first, that exactly this file is
# not already registered. In case it is, the file will be rejected.
regPi = srvObj.getCfg().register_plugins[mimeType]
logger.debug("Plugin found for %s: %s", mimeType, regPi)
params = ngamsPlugInApi.parseRawPlugInPars(regPi.pars)
tmpReqPropsObj = ngamsReqProps.ngamsReqProps().\
setMimeType(mimeType).\
setStagingFilename(filename).\
setTargDiskInfo(diskInfoDic[diskId]).\
setHttpMethod(NGAMS_HTTP_GET).\
setCmd(NGAMS_REGISTER_CMD).\
setSize(os.path.getsize(filename)).\
setFileUri(filename).\
setNoReplication(1)
tmpFileObj = ngamsFileInfo.ngamsFileInfo()
try:
# Invoke Registration Plug-In.
piName = regPi.name
plugInMethod = loadPlugInEntryPoint(piName)
piRes = plugInMethod(srvObj, tmpReqPropsObj, params)
del tmpReqPropsObj
# Check if this file is already registered on this disk. In case
# yes, it is not registered again.
files = srvObj.db.getFileSummary1(srvObj.getHostId(),
[piRes.getDiskId()],
[piRes.getFileId()])
fileRegistered = 0
for tmpFileInfo in files:
tmpMtPt = tmpFileInfo[ngamsDbCore.SUM1_MT_PT]
tmpFilename = tmpFileInfo[ngamsDbCore.SUM1_FILENAME]
tmpComplFilename = os.path.normpath(tmpMtPt + "/" +\
tmpFilename)
if (tmpComplFilename == filename):
fileRegistered = 1
break
if (fileRegistered):
fileRejectCount += 1
tmpMsgForm = "REJECTED: File with File ID/Version: %s/%d " +\
"and path: %s is already registered on disk " +\
"with Disk ID: %s"
tmpMsg = tmpMsgForm % (piRes.getFileId(),
piRes.getFileVersion(), filename,
piRes.getDiskId())
logger.warning(tmpMsg + ". File is not registered again.")
if (emailNotif):
tmpFileObj.\
setDiskId(diskId).setFilename(filename).\
setTag(tmpMsg)
regDbm.addIncKey(tmpFileObj)
if (reqPropsObj):
reqPropsObj.incActualCount(1)
ngamsHighLevelLib.stdReqTimeStatUpdate(
srvObj, reqPropsObj, regTimeAccu)
regTimeAccu += time.time() - reg_start
fileCount += 1
continue
# Calculate checksum. We maintain the old name for backwards
# compatibility
crc_variant = srvObj.cfg.getCRCVariant()
if crc_variant == ngamsFileUtils.CHECKSUM_CRC32_INCONSISTENT:
crc_variant = 'ngamsGenCrc32'
checksum = ngamsFileUtils.get_checksum(65536, filename,
crc_variant) or ''
# Move file and update information about file in the NGAS DB.
mvFile(filename, piRes.getCompleteFilename())
ngamsArchiveUtils.updateFileInfoDb(srvObj, piRes, checksum,
crc_variant)
ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), piRes)
ngamsLib.makeFileReadOnly(piRes.getCompleteFilename())
if (emailNotif):
uncomprSize = piRes.getUncomprSize()
ingestDate = time.time()
creDateSecs = getFileCreationTime(filename)
tmpFileObj.\
setDiskId(diskId).\
setFilename(filename).\
setFileId(piRes.getFileId()).\
setFileVersion(piRes.getFileVersion()).\
setFormat(piRes.getFormat()).\
setFileSize(piRes.getFileSize()).\
setUncompressedFileSize(uncomprSize).\
setCompression(piRes.getCompression()).\
setIngestionDate(ingestDate).\
setIgnore(0).\
setChecksum(checksum).\
setChecksumPlugIn(crc_variant).\
setFileStatus(NGAMS_FILE_STATUS_OK).\
setCreationDate(creDateSecs).\
setTag("REGISTERED")
fileRegCount += 1
# If running as a cache archive, update the Cache New Files DBM
# with the information about the new file.
if (srvObj.getCachingActive()):
fileVer = fio.getFileVersion()
ngamsCacheControlThread.addEntryNewFilesDbm(
srvObj, diskId, piRes.getFileId(), fileVer, filename)
# Generate a confirmation log entry.
msg = genLog("NGAMS_INFO_FILE_REGISTERED", [
filename,
piRes.getFileId(),
piRes.getFileVersion(),
piRes.getFormat()
])
time.sleep(0.005)
regTime = time.time() - reg_start
msg = msg + ". Time: %.3fs." % (regTime)
logger.info(msg, extra={'to_syslog': 1})
except Exception as e:
errMsg = genLog("NGAMS_ER_FILE_REG_FAILED", [filename, str(e)])
logger.error(errMsg)
if (emailNotif):
tmpFileObj.\
setDiskId(diskId).setFilename(filename).\
setTag(errMsg)
fileFailCount += 1
regTime = time.time() - reg_start
# TODO (rtobar, 2016-01): Why don't we raise an exception here?
# Otherwise the command appears as successful on the
# client-side
# Add the file information in the registration report.
if (emailNotif): regDbm.addIncKey(tmpFileObj)
# Update request status time information.
regTimeAccu += regTime
if (reqPropsObj):
reqPropsObj.incActualCount(1)
ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj,
regTimeAccu)
fileCount += 1
fo.close()
rmFile(sortFileList)
if (emailNotif): regDbm.sync()
del fileListDbm
rmFile(fileListDbmName + "*")
# Final update of the Request Status.
if (reqPropsObj):
if (reqPropsObj.getExpectedCount() and reqPropsObj.getActualCount()):
complPercent = (100.0 * (float(reqPropsObj.getActualCount()) /
float(reqPropsObj.getExpectedCount())))
else:
complPercent = 100.0
reqPropsObj.setCompletionPercent(complPercent, 1)
reqPropsObj.setCompletionTime(1)
srvObj.updateRequestDb(reqPropsObj)
# Send Register Report with list of files cloned to a possible
# requestor(select) of this.
if (emailNotif):
xmlStat = 0
if (xmlStat):
# Create an instance of the File List Class, used to store the
# Registration Report.
regStat = ngamsFileList.\
ngamsFileList("FILE_REGISTRATION_STATUS",
"Status report for file " +\
"registration")
# Loop over the file objects in the BSD DB and add these
# in the status object.
regDbm.initKeyPtr()
while (1):
key, tmpFileObj = regDbm.getNext()
if (not key): break
regStat.addFileInfoObj(tmpFileObj)
# Set overall status of registration procedure.
regStat.setStatus("Files Found: " + str(fileCount + 1) + ", "+\
"Files Registered: " + str(fileRegCount) +\
", " +\
"Files Failed: " + str(fileFailCount) + ", " +\
"Files Rejected: " + str(fileRejectCount))
status = srvObj.genStatus(NGAMS_SUCCESS,
"REGISTER command status report").\
addFileList(regStat)
statRep = status.genXmlDoc()
statRep = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, statRep)
mimeType = NGAMS_XML_MT
else:
# Generate a 'simple' ASCII report.
statRep = tmpFilePat + "_NOTIF_EMAIL.txt"
fo = open(statRep, "w")
if (reqPropsObj):
path = reqPropsObj.getHttpPar("path")
else:
path = "-----"
if (fileCount):
timePerFile = (regTimeAccu / fileCount)
else:
timePerFile = 0
tmpFormat = "REGISTER STATUS REPORT:\n\n" +\
"==Summary:\n\n" +\
"Date: %s\n" +\
"NGAS Host: %s\n" +\
"Search Path: %s\n" +\
"Total Number of Files: %d\n" +\
"Number of Registered Files: %d\n" +\
"Number of Failed Files: %d\n" +\
"Number of Rejected Files: %d\n" +\
"Total processing time (s): %.3f\n" +\
"Handling time per file (s): %.3f\n\n" +\
"==File List:\n\n"
fo.write(tmpFormat % (toiso8601(), srvObj.getHostId(), path,
fileCount, fileRegCount, fileFailCount,
fileRejectCount, regTimeAccu, timePerFile))
tmpFormat = "%-80s %-32s %-3s %-10s\n"
fo.write(tmpFormat %\
("Filename", "File ID", "Ver", "Status"))
fo.write(tmpFormat % (80 * "-", 32 * "-", 3 * "-", 10 * "-"))
regDbm.initKeyPtr()
while (1):
key, tmpFileObj = regDbm.getNext()
if (not key): break
mtPt = diskInfoDic[tmpFileObj.getDiskId()].getMountPoint()
filename = os.path.normpath(mtPt + "/" +\
tmpFileObj.getFilename())
line = tmpFormat %\
(filename, tmpFileObj.getFileId(),
str(tmpFileObj.getFileVersion()),
tmpFileObj.getTag())
fo.write(line)
fo.write(128 * "-")
fo.write("\n\n==END\n")
fo.close()
mimeType = NGAMS_TEXT_MT
# Send out the status report.
emailAdrList = reqPropsObj.getHttpPar("notif_email").split(",")
attachmentName = "RegisterStatusReport"
if (reqPropsObj.hasHttpPar("path")):
attachmentName += "-" + reqPropsObj.getHttpPar("path").\
replace("/", "_")
ngamsNotification.notify(srvObj.host_id,
srvObj.cfg,
NGAMS_NOTIF_INFO,
"REGISTER STATUS REPORT",
statRep,
recList=emailAdrList,
force=1,
contentType=mimeType,
attachmentName=attachmentName)
del regDbm
rmFile(regDbmName + "*")
rmFile(statRep)
# Generate final status log + exit.
if (fileCount > 0):
timePerFile = (regTimeAccu / fileCount)
else:
timePerFile = 0.0
msg = "Registration procedure finished processing Register Request - " + \
"terminating. Files handled: %d. Total time: %.3fs. " + \
"Average time per file: %.3fs."
logger.debug(msg, fileCount, regTimeAccu, timePerFile)
def register(srvObj, path, mimeType="", reqPropsObj=None, httpRef=None):
"""
Function to generate a list of candidate files to register, and to
launch a thread that actually carries out the registration.
When the possible candidate files haven been selected, a reply is sent
back to the requestor if the 'httpRef' object is given.
srvObj: Instance of NG/AMS Server object (ngamsServer).
path: The path name, which is used as starting point for
the searching for files (string).
mimeType: Comma separated list of mime-types, which should be
considered for registration (string).
reqPropsObj: If an NG/AMS Request Properties Object is given, the
Request Status will be updated as the request is carried
out (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
# Check if the given path or file exists.
if (not os.path.exists(path)):
errMsg = genLog("NGAMS_ER_FILE_REG_FAILED",
[path, "Non-existing file or path."])
raise Exception(errMsg)
# Check if the given path/file is on one of the NGAS Disks.
foundPath = 0
for slotId in srvObj.getDiskDic().keys():
do = srvObj.getDiskDic()[slotId]
if (path.find(do.getMountPoint()) == 0):
foundPath = 1
break
if (not foundPath):
errMsg = genLog("NGAMS_ER_FILE_REG_FAILED", [
path, "File or path specified is not located on an "
"NGAS Disk."
])
raise Exception(errMsg)
# Create file pattern for temporary files.
tmpFilePat = ngamsHighLevelLib.genTmpFilename(srvObj.getCfg(),
"REGISTER_CMD")
# Generate dictionary with mime-types to accept.
mimeTypeDic = {}
if (mimeType != ""):
for mt in mimeType.split(","):
try:
mimeTypeDic[mt] = srvObj.getMimeTypeDic()[mt]
except:
errMsg = genLog("NGAMS_ER_REQ_HANDLING",
["Mime-type specified: " + mt + " " +\
"in connection with REGISTER command " +\
"does not have a DAPI defined"])
raise Exception(errMsg)
else:
mimeTypeDic = srvObj.getMimeTypeDic()
# From the Disk Dictionary, generate a dictionary with mappings from
# Disk ID + from Mount Point to the disk information for the disk.
diskInfoDic = {}
mtPt2DiskInfo = {}
for slotId in srvObj.getDiskDic().keys():
if (not srvObj.getCfg().getSlotIdDefined(slotId)): continue
diskId = srvObj.getDiskDic()[slotId].getDiskId()
mtPt = srvObj.getDiskDic()[slotId].getMountPoint()
tmpDiskInfo = ngamsDiskInfo.ngamsDiskInfo().\
read(srvObj.getDb(), diskId)
diskInfoDic[diskId] = tmpDiskInfo
mtPt2DiskInfo[mtPt] = tmpDiskInfo
# Generate a list with all files found under the specified path, which
# are candidates for being registered.
fileListDbmName = tmpFilePat + "_FILE_LIST"
rmFile(fileListDbmName + "*")
fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm=1)
mimeTypeMappings = srvObj.getCfg().getMimeTypeMappings()
tmpGlobFile = tmpFilePat + "_FILE_GLOB.glob"
fileCount = 0
searchPath = os.path.normpath(path)
foundVolume = False
for mtPt in mtPt2DiskInfo.keys():
mtPt2 = mtPt
if (mtPt[-1] != "/"): mtPt2 += "/"
if (searchPath.find(mtPt2) == 0):
foundVolume = True
if os.path.isdir(searchPath):
for root, dirs, files in os.walk(searchPath):
for nextFile in files:
if nextFile not in \
[NGAMS_DISK_INFO, \
NGAMS_VOLUME_ID_FILE, \
NGAMS_VOLUME_INFO_FILE]:
mimeType = ngamsLib.detMimeType(
mimeTypeMappings, nextFile, 0)
tmpFileInfo = [
os.path.join(root, nextFile),
mtPt2DiskInfo[mtPt].getDiskId(), mimeType
]
fileListDbm.add(os.path.join(root, nextFile),
tmpFileInfo)
elif os.path.isfile(
searchPath): # the path is actually pointing to a file
nextFile = os.path.basename(searchPath)
root = os.path.dirname(searchPath)
if nextFile not in \
[NGAMS_DISK_INFO, \
NGAMS_VOLUME_ID_FILE, \
NGAMS_VOLUME_INFO_FILE]:
mimeType = ngamsLib.detMimeType(mimeTypeMappings, nextFile,
0)
tmpFileInfo = [
os.path.join(root, nextFile),
mtPt2DiskInfo[mtPt].getDiskId(), mimeType
]
fileListDbm.add(os.path.join(root, nextFile), tmpFileInfo)
if foundVolume: break
# pattern = ""
# # TODO: Portatibility issue. The usage of UNIX commands should be
# # avoided if possible.
# while (1):
# # Use a shell commands here to avoid building up maybe huge lists in
# # memory/in the Python interpreter. A better way could maybe be found
# # avoiding to invoke a shell command.
# rmFile(tmpGlobFile)
# searchPath = os.path.normpath(path + pattern)
# tmpCmd = "find " + searchPath + " -maxdepth 1 > " + tmpGlobFile
# stat, out = commands.getstatusoutput(tmpCmd)
# if (stat != 0): break
# fo = open(tmpGlobFile)
# while (1):
# nextFile = fo.readline().strip()
# if (nextFile == ""): break
# if (nextFile.find("/" + NGAMS_DISK_INFO) != -1): continue
# if (nextFile.find("/" + NGAMS_VOLUME_ID_FILE) != -1): continue
# if (nextFile.find("/" + NGAMS_VOLUME_INFO_FILE) != -1): continue
# if (os.path.isfile(nextFile)):
# nextFile = os.path.normpath(nextFile)
#
# # Find the host disk. Only files located on mounted NGAS
# # disks are considered.
# for mtPt in mtPt2DiskInfo.keys():
# mtPt2 = mtPt
# if (mtPt[-1] != "/"): mtPt2 += "/"
# if (nextFile.find(mtPt2) == 0):
# info(4,"Found candidate file for registering: " +\
# nextFile)
# # Take only files with a Registration Plug-In defined.
# mimeType = ngamsLib.detMimeType(mimeTypeMappings,
# nextFile, 0)
# regPi = srvObj.getCfg().getRegPiFromMimeType(mimeType)
# if (regPi != None):
# tmpFileInfo = [nextFile,
# mtPt2DiskInfo[mtPt].getDiskId(),
# mimeType]
# fileListDbm.add(nextFile, tmpFileInfo)
# break
# fo.close()
# pattern += "/*"
# rmFile(tmpGlobFile)
fileListDbm.sync()
del fileListDbm
# Send intermediate reply if the HTTP Reference object is given.
is_async = 'async' in reqPropsObj and int(reqPropsObj['async'])
if httpRef and is_async:
logger.debug("REGISTER command accepted - generating immediate " +\
"confimation reply to REGISTER command")
# Update the request status in the Request Properties Object.
reqPropsObj.\
setExpectedCount(fileCount).\
setActualCount(0).setCompletionPercent(0)
srvObj.updateRequestDb(reqPropsObj)
status = srvObj.genStatus(NGAMS_SUCCESS,
"Accepted REGISTER command for execution").\
setReqStatFromReqPropsObj(reqPropsObj).\
setActualCount(0)
# Launch the register thread or run the command in foreground if wait=1
if is_async:
args = (srvObj, fileListDbmName, tmpFilePat, diskInfoDic, reqPropsObj,
None)
thrName = NGAMS_REGISTER_THR + threading.current_thread().getName()
regThread = threading.Thread(None, _registerThread, thrName, args)
regThread.setDaemon(0)
regThread.start()
else:
# Carry out the REGISTER Command (directly in this thread) and send
# reply when this is done.
_registerExec(srvObj, fileListDbmName, tmpFilePat, diskInfoDic,
reqPropsObj)
msg = "Successfully handled command REGISTER"
logger.debug(msg)
status = srvObj.genStatus(NGAMS_SUCCESS, msg).\
setReqStatFromReqPropsObj(reqPropsObj).setActualCount(0)
# Send reply if possible.
if (httpRef):
xmlStat = status.genXmlDoc(0, 0, 0, 1, 0)
xmlStat = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, xmlStat)
httpRef.send_data(six.b(xmlStat), NGAMS_XML_MT)
logger.debug("Querying total DB time")
msg = "Total DB time: %.6fs" % srvObj.getDb().getDbTime()
elif (dbTimeReset):
msg = "Resetting DB timer"
logger.debug(msg)
srvObj.getDb().resetDbTime()
else:
msg = "Successfully handled command STATUS"
if (reqPropsObjRef == reqPropsObj):
reqPropsObj.setCompletionTime()
srvObj.updateRequestDb(reqPropsObj)
if (genCfgStatus or genDiskStatus or genFileStatus or genRequestStatus):
status.setReqStatFromReqPropsObj(reqPropsObjRef)
# Generate XML reply.
xmlStat = status.genXmlDoc(genCfgStatus, genDiskStatus, genFileStatus,
genStatesStatus)
xmlStat = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, xmlStat)
httpRef.send_data(xmlStat, NGAMS_XML_MT)
elif not httpRef.reply_sent:
httpRef.send_status(msg)
if (msg and (not help)):
logger.info(msg)
else:
logger.info("Successfully handled command STATUS")
# EOF
