def checkFilesRemovable(srvObj, fileListDbmName, tmpFilePat):
"""
Check for each file represented by a File Info Object if it can be
deleted/removed from the disk.
srvObj: Reference to NG/AMS server class object (ngamsServer).
fileListDbmName: Name of DBM DB containing the information about the
files in question (string
tmpFilePat: Pattern used to generate temporary filenames (string).
Returns: Name of DBM DB with information about files that
cannot be removed (string).
"""
# DBM DB containing information about non-removable files.
filename = "_NON_REMOVABLE_FILES"
nonRemFilesDbmName = os.path.normpath(tmpFilePat + "_NON_REMOVABLE_FILES")
nonRemFilesDbm = ngamsDbm.ngamsDbm(nonRemFilesDbmName, writePerm=1)
fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName)
fileListDbm.initKeyPtr()
while (1):
key, fileInfo = fileListDbm.getNext()
if (not key): break
mtPt = fileInfo[ngamsDbCore.SUM1_MT_PT]
fileName = fileInfo[ngamsDbCore.SUM1_FILENAME]
complFilename = os.path.normpath(mtPt + "/" + fileName)
if (not ngamsLib.fileRemovable(complFilename)):
nonRemFilesDbm.addIncKey(fileInfo)
del nonRemFilesDbm
del fileListDbm
return nonRemFilesDbmName
def recreate_dbm(self):
try:
os.unlink(self.dbm_fname)
except OSError as e:
if e.errno != errno.ENOENT:
raise
self.dbm = ngamsDbm.ngamsDbm(self.dbm_fname, cleanUpOnDestr=0, writePerm=1)
def _handleFileList(srvObj, reqPropsObj, httpRef):
"""
Handle STATUS?file_list... Command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: The File List ID allocated for this request (string).
"""
T = TRACE()
# Should a lower limit for the ingestion date be taken into account.
fromIngDate = None
if (reqPropsObj.hasHttpPar("from_ingestion_date")):
tmpTromIngDate = reqPropsObj.getHttpPar("from_ingestion_date")
fromIngDate = fromiso8601(tmpTromIngDate)
# Handle the unique flag. If this is specified, only information for unique
# File ID/Version pairs are returned.
unique = False
if (reqPropsObj.hasHttpPar("unique")):
unique = int(reqPropsObj.getHttpPar("unique"))
# Dump the file information needed.
fileListId = genUniqueId()
dbmBaseName = STATUS_FILE_LIST_DBM_TAG % fileListId
fileInfoDbmName = ngamsHighLevelLib.genTmpFilename(srvObj.getCfg(),
dbmBaseName)
# Dump the file info from the DB. Deal with uniqueness quickly
# (instead of using a new file like before)
try:
fileInfoDbm = ngamsDbm.ngamsDbm(fileInfoDbmName, 0, 1)
fileCount = 1
unique_files = set()
for f in srvObj.db.files_in_host(srvObj.getHostId(),
from_date=fromIngDate):
if unique:
key = str('%s_%d' % (f[2], f[3]))
if key in unique_files:
continue
else:
key = str(fileCount)
fileInfoDbm.add(key, f)
fileCount += 1
except Exception as e:
rmFile(fileInfoDbmName)
msg = "Problem generating file list for STATUS Command. " +\
"Parameters: from_ingestion_date=%s. Error: %s" %\
(str(fromIngDate), str(e))
raise Exception(msg)
return fileListId
def checkSpuriousFiles(srvObj,
tmpFilePat=None,
hostId=None,
diskId=None,
fileId=None,
fileVersion=None):
"""
Check if there are any spurious files in the DB either marked as to be
ignored or having a status indicating a problem. If such are found
according to the criterias defined, these are added in a DBM DB, which
name is returned.
srvObj: Reference to NG/AMS server class object (ngamsServer).
tmpFilePat: Pattern to apply for temporary files (string).
hostId: Name of NGAS host on which the files reside (string).
diskId: Disk ID of disk to take into account (string|None).
fileId: File ID of file(s) to take into account (string|None).
fileVersion: Version of file(s) to take into account (integer|None).
Returns: Returns name of DBM DB with references to spurious
files found (string).
"""
T = TRACE()
if (hostId == ""): hostId = None
if (diskId == ""): diskId = None
if (fileId == ""): fileId = None
if (fileVersion == -1): fileVersion = None
# DBM DB containing information about spurious files.
filename = "_SPURIOUS_FILES"
if (tmpFilePat):
spuriousFilesDbmName = os.path.normpath(tmpFilePat + filename)
else:
spuriousFilesDbmName = ngamsHighLevelLib.\
genTmpFilename(srvObj.getCfg(), filename)
spuriousFilesDbm = ngamsDbm.ngamsDbm(spuriousFilesDbmName, writePerm=1)
# Check that there are no spurious files in connection with this disk in
# the DB (where ngas_files.file_ignore != 0 or ngas_files.status != "1*******"
files = srvObj.db.getFileSummarySpuriousFiles1(hostId,
diskId,
fileId,
fileVersion,
fetch_size=200)
for fileInfo in files:
spuriousFilesDbm.addIncKey(fileInfo)
spuriousFilesDbm.sync()
del spuriousFilesDbm
return spuriousFilesDbmName
def checkSystem(testParDic):
"""
Carry out a system check for the given system
testParDic: Dictionary with parameters for running the test (dictionary).
Returns: Void.
"""
info(4, "Entering serverLoop() ...")
checkCreatePath(testParDic["WORKING-DIR"])
srvList = testParDic["SERVERS"].split(",")
srvList.sort()
if (testParDic["IGNORE-SERVERS"]):
ignoreSrvList = testParDic["IGNORE-SERVERS"].split(",")
ignoreSrvList.sort()
else:
ignoreSrvList = []
domainList = testParDic["DOMAINS"].split(",")
domainList.sort()
dbSrv, db, user, password = ngasUtilsLib.getDbPars()
dbCon = ngamsDb.ngamsDb(dbSrv, db, user, password, 0)
srvStr = str(srvList)[1:-1].replace(",", "-").replace(" ", "").\
replace("'", "")
domStr = str(domainList)[1:-1].replace(",", "-").replace(" ", "").\
replace("'", "")
dbmName = os.path.normpath("%s/NGAS_SYS_CHECK_%s_%s_%s.bsddb" %\
(testParDic["WORKING-DIR"], os.environ["USER"],
srvStr, domStr))
dbmObj = ngamsDbm.ngamsDbm(dbmName, writePerm=1)
testParDic["_testTmpPath"] = os.path.normpath("%s/NGAS_SYS_CHECK_%.6f" %\
(testParDic["WORKING-DIR"],
time.time()))
os.system("mkdir -p %s" % testParDic["_testTmpPath"])
try:
serverLoop(testParDic, srvList, ignoreSrvList, domainList, dbCon,
dbmObj)
testParDic["_endTime"] = time.time()
dbmObj.sync()
os.system("rm -rf %s" % testParDic["_testTmpPath"])
except Exception, e:
dbmObj.sync()
os.system("rm -rf %s" % testParDic["_testTmpPath"])
raise Exception, e
def _remFile(srvObj, reqPropsObj, diskId, fileId, fileVersion, execute,
tmpFilePat):
"""
See documentation for the ngamsRemFileCmd.remFile() function.
"""
# Check for illegal parameter combinations.
if ((not diskId) or (diskId and (not fileId))):
errMsg = "Disk ID: %s, File ID: %s, File Version: %d" %\
(diskId, fileId, fileVersion)
errMsg = genLog("NGAMS_ER_CMD_SYNTAX", [NGAMS_REMFILE_CMD, errMsg])
raise Exception(errMsg)
# Temporary DBM to contain SQL info about files concerned by the query.
fileListDbmName = os.path.normpath(tmpFilePat + "_FILE_LIST")
fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm=1)
# Get the information from the DB about the files in question.
hostId = None
diskIds = []
fileIds = []
if (diskId):
diskIds = [diskId]
elif ((not diskId) and fileId):
hostId = srvObj.getHostId()
if (fileId): fileIds = [fileId]
if (fileVersion == -1): fileVersion = None
n_files = 0
for f in srvObj.db.getFileSummary1(hostId, diskIds, fileIds, ignore=None):
if fileVersion is not None and fileVersion != f[
ngamsDbCore.SUM1_VERSION]:
continue
msg = "Scheduling file with ID: %s/%d on disk with ID: %s for " +\
"deletion"
logger.debug(msg, f[ngamsDbCore.SUM1_FILE_ID],
f[ngamsDbCore.SUM1_VERSION], f[ngamsDbCore.SUM1_DISK_ID])
fileListDbm.add(str(n_files), f)
n_files += 1
fileListDbm.sync()
# Check if the files selected for deletion are available within the NGAS
# system, in at least 3 copies.
filesMisCopyDbmName, filesNotRegDbmName, complFileListDbmName =\
ngamsRemUtils.checkFileCopiesAndReg(srvObj,
3, tmpFilePat,
fileListDbmName)
status = ngamsRemUtils._remStatErrReport(srvObj, reqPropsObj, tmpFilePat,
filesMisCopyDbmName,
filesNotRegDbmName,
fileListDbmName, diskId, fileId,
fileVersion)
if (status): return status
# Check that none of the matching files are stored on other NGAS Nodes.
# If such are found, do not consider these.
fileListDbm.initKeyPtr()
remKeyList = []
while (1):
key, fileInfo = fileListDbm.getNext()
if (not key): break
if (fileInfo[ngamsDbCore.SUM1_HOST_ID] != srvObj.getHostId()):
remKeyList.append(key)
for remKey in remKeyList:
fileListDbm.rem(remKey)
#########################################################################
# Execute the deletion if execute = 1 and files were found to be deleted.
#########################################################################
successDelCount = 0
failedDelCount = 0
if (execute):
fileListDbm.initKeyPtr()
run = 1
# TODO: This should be changed to a single or a few DB transactions for all files
# and a bulk rm for the same number of files.
while (run):
key, fileInfo = fileListDbm.getNext()
if (not key):
run = 0
continue
try:
diskId = fileInfo[ngamsDbCore.SUM1_DISK_ID]
fileId = fileInfo[ngamsDbCore.SUM1_FILE_ID]
fileVer = fileInfo[ngamsDbCore.SUM1_VERSION]
mtPt = fileInfo[ngamsDbCore.SUM1_MT_PT]
filename = fileInfo[ngamsDbCore.SUM1_FILENAME]
complFilename = os.path.normpath(mtPt + "/" + filename)
msg = "Deleting DB info for file: %s/%s/%d"
logger.debug(msg, diskId, fileId, fileVer)
# We remove first the DB info and afterwards the file on the
# disk. The reason for this is that it is considered worse
# to have an entry for a file in the DB, which is not on disk
# than vice versa, since NGAS uses the info in the DB to check
# for the number of available copies.
try:
srvObj.getDb().deleteFileInfo(srvObj.getHostId(), diskId,
fileId, fileVer)
infoMsg = genLog("NGAMS_INFO_DEL_FILE",
[diskId, fileId, fileVer])
logger.debug(infoMsg)
successDelCount += 1
except Exception, e:
failedDelCount += 1
errMsg = genLog("NGAMS_ER_DEL_FILE_DB",
[diskId, fileId, fileVer,
str(e)])
logger.warning(errMsg)
# Removing the DB info was successful, remove the copy on disk.
msg = "Deleting copy of file: %s/%s/%d: %s"
logger.debug(msg, diskId, fileId, fileVer, complFilename)
rmFile(complFilename)
except Exception, e:
failedDelCount += 1
errMsg = genLog(
"NGAMS_ER_DEL_FILE_DISK",
[diskId, fileId, fileVer, str(e)])
logger.warning(errMsg)
def _clone(srvObj, diskId, fileId, fileVersion, targetDiskId, reqPropsObj,
httpRef, tmpFilePat):
"""
Internal function used by ngamsCloneCmd.clone() to carry out the
cloning. See documentation for ngamsCloneCmd.clone().
"""
targetDiskId = targetDiskId.strip()
logger.debug("Handling file cloning with parameters - File ID: %s -" + \
"Disk ID: %s - File Version: %s - Target Disk ID: |%s|",
fileId, diskId, str(fileVersion), targetDiskId)
if (((fileId == "") and (diskId == "") and (fileVersion != -1))
or ((fileId == "") and (diskId == "") and (fileVersion == -1))):
errMsg = genLog("NGAMS_ER_CMD_SYNTAX",
[NGAMS_CLONE_CMD, "File Id: " + fileId +\
", Disk ID: " + diskId +\
", File Version: " + str(fileVersion)])
raise Exception, errMsg
# If Disk ID, File ID and File Version are given, execute a quick cloning.
try:
fileVersion = int(fileVersion)
except:
pass
if (False and diskId and fileId and (fileVersion > 0)):
_cloneExplicit(srvObj, reqPropsObj, diskId, fileId, fileVersion,
targetDiskId)
logger.info("Successfully handled command CLONE")
return
# Handling cloning of more files.
cloneListDbm = None
cloneListDbmName = tmpFilePat + "_CLONE_INFO_DB"
try:
# Get information about candidate files for cloning.
files = srvObj.db.getFileInfoFromFileId(fileId,
fileVersion,
diskId,
ignore=0,
order=0,
dbCursor=False)
if not files:
msg = genLog("NGAMS_ER_CMD_EXEC",
[NGAMS_CLONE_CMD, "No files for cloning found"])
raise Exception(msg)
# Convert to tuple of file info object plus extra info
# This is how the code expects this information to come, so we need to
# keep the format unless we change the bulk of the code
# f[-2] is the host id, f[-1] is the mount point
all_info = []
for f in files:
all_info.append((ngamsFileInfo.ngamsFileInfo().unpackSqlResult(f),
f[-2], f[-1]))
# Create a BSD DB with information about files to be cloned.
rmFile(cloneListDbmName + "*")
cloneListDbm = ngamsDbm.ngamsDbm(cloneListDbmName,
cleanUpOnDestr=0,
writePerm=1)
cloneDbCount = 0
if fileId != "" and (diskId != "" or fileVersion == -1):
# Take only the first element
cloneListDbm.add("0", all_info[0])
cloneDbCount = 1
else:
# Take all the files.
for tmpFileInfo in all_info:
cloneListDbm.add(str(cloneDbCount), tmpFileInfo)
cloneDbCount += 1
except Exception:
if (cloneListDbm): del cloneListDbm
rmFile(cloneListDbmName + "*")
raise
logger.debug("Found: %d file(s) for cloning ...", cloneDbCount)
del cloneListDbm
# Check available amount of disk space.
cloneCheckDiskSpace(srvObj, cloneListDbmName, tmpFilePat, targetDiskId)
# Initialize Request Status parameters.
if (reqPropsObj):
reqPropsObj.\
setCompletionPercent(0, 1).\
setExpectedCount(cloneDbCount, 1).\
setActualCount(0, 1)
srvObj.updateRequestDb(reqPropsObj)
# Wait until CLONE Command has finished, or send a reply before cloning?
async = 'async' in reqPropsObj and int(reqPropsObj['async'])
if async:
# Send intermediate reply if the HTTP Reference object is given
# whenever send an auto reply now.
logger.debug("CLONE command accepted - generating immediate " +
"confimation reply to CLONE command")
status = srvObj.genStatus(NGAMS_SUCCESS,
"Accepted CLONE command for execution").\
setReqStatFromReqPropsObj(reqPropsObj).\
setActualCount(0)
# Do the actual cloning in a thread
args = (srvObj, cloneListDbmName, tmpFilePat, targetDiskId,
reqPropsObj, None)
thrName = NGAMS_CLONE_THR + threading.current_thread().getName()
cloneThread = threading.Thread(None, _cloneThread, thrName, args)
cloneThread.setDaemon(0)
cloneThread.start()
else:
# Carry out the cloning (directly in this thread) and send reply
# when this is done.
_cloneExec(srvObj, cloneListDbmName, tmpFilePat, targetDiskId,
reqPropsObj)
msg = "Successfully handled command CLONE"
logger.debug(msg)
status = srvObj.genStatus(NGAMS_SUCCESS, msg).\
setReqStatFromReqPropsObj(reqPropsObj).setActualCount(0)
rmFile(cloneListDbmName + "*")
# Send reply if possible.
if (httpRef):
xmlStat = status.genXmlDoc(0, 0, 0, 1, 0)
xmlStat = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, xmlStat)
httpRef.send_data(xmlStat, NGAMS_XML_MT)
def _cloneExec(srvObj, cloneListDbmName, tmpFilePat, targetDiskId,
reqPropsObj):
"""
See documentation of ngamsCloneCmd._cloneThread(). This function is
merely implemented in order to encapsulate the whole process to be able
to clean up properly when the processing is terminated.
"""
T = TRACE(1)
cloneListDbm = cloneStatusDbm = None
emailNotif = 0
checkChecksum = 1
if (reqPropsObj):
if (reqPropsObj.hasHttpPar("notif_email")): emailNotif = 1
if (reqPropsObj.hasHttpPar("check")):
checkChecksum = int(reqPropsObj.getHttpPar("check"))
# Open clone list DB.
cloneListDbm = ngamsDbm.ngamsDbm(cloneListDbmName)
# We have to get the port numbers of the hosts where the files to be
# cloned are stored.
hostInfoDic = {}
cloneListDbm.initKeyPtr()
while (1):
key, fileInfo = cloneListDbm.getNext()
if (not key): break
hostInfoDic[fileInfo[1]] = -1
hostInfoDic = ngamsHighLevelLib.resolveHostAddress(srvObj.getHostId(),
srvObj.getDb(),
srvObj.getCfg(),
hostInfoDic.keys())
# The cloning loop. Loop over the list of files to clone and generate
# a report with the result.
if (emailNotif):
cloneStatusDbmName = tmpFilePat + "_CLONE_STATUS_DB"
cloneStatusDbm = ngamsDbm.ngamsDbm(cloneStatusDbmName,
cleanUpOnDestr=0,
writePerm=1)
successCloneCount = 0
failedCloneCount = 0
abortCloneLoop = 0
timeAccu = 0.0
key = 0
while (1):
clone_start = time.time()
if (not cloneListDbm.hasKey(str(key))): break
fileInfo = cloneListDbm.get(str(key))
key += 1
# Check if we have permission to run. Otherwise, stop.
if (not srvObj.getThreadRunPermission()): break
fio = fileInfo[0]
mtPt = fileInfo[2]
if (emailNotif):
tmpFileList = ngamsFileList.\
ngamsFileList("FILE_CLONE_STATUS",
"File: " + fio.getFileId() + "/" +\
fio.getDiskId() + "/" +\
str(fio.getFileVersion()))
hostId = fileInfo[1]
text = "Cloning file - File ID: %s/%d, on disk " +\
"with ID: %s on host: %s"
logger.debug(text, fio.getFileId(), fio.getFileVersion(),
fio.getDiskId(), hostId)
# We generate a local Staging File and archive this.
stagingFilename = ""
try:
# Check if file is marked as bad.
if (fio.getFileStatus()[0] == "1"):
errMsg = "File marked as bad - skipping!"
raise Exception, errMsg
if (targetDiskId == ""):
# Try to find a disk not hosting already a file with that
# ID + version.
diskExemptList = [fio.getDiskId()]
while (1):
trgDiskInfo = ngamsDiskUtils.\
findTargetDisk(srvObj.getHostId(),
srvObj.getDb(),
srvObj.getCfg(),
fio.getFormat(),
1, diskExemptList)
# Check if a file with that ID + version is already
# stored on the selected Target Disk.
if (srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(),
fio.getFileId(),
fio.getFileVersion())):
# This file is already stored on the given disk.
# Add to the exempt list.
diskExemptList.append(trgDiskInfo.getDiskId())
else:
# OK, this disk should be OK, stop looking for a
# suitable Target Disk.
break
else:
try:
trgDiskInfo = ngamsDiskInfo.ngamsDiskInfo().\
read(srvObj.getDb(), targetDiskId)
slotId = trgDiskInfo.getSlotId()
storageSetId = srvObj.getCfg().\
getStorageSetFromSlotId(slotId).\
getStorageSetId()
trgDiskInfo.setStorageSetId(storageSetId)
except Exception:
abortCloneLoop = 1
raise
# We don't accept to clone onto the same disk (this would mean
# overwriting).
if (trgDiskInfo.getDiskId() == fio.getDiskId()):
err = "Source and target files are identical"
msg = "Failed in cloning file with ID: " + fio.getFileId() +\
"/Version: " + str(fio.getFileVersion()) +\
" on disk with ID: " + fio.getDiskId() +\
" on host: " + hostId + ". Reason: " + err
logger.warning(msg)
if (emailNotif):
tmpFileList.setStatus(NGAMS_FAILURE + ": " + err)
tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE"))
cloneStatusDbm.addIncKey(tmpFileList)
failedCloneCount += 1
continue
tmpReqPropsObj = ngamsReqProps.ngamsReqProps()
tmpReqPropsObj.setMimeType(fio.getFormat())
stagingFilename = ngamsHighLevelLib.\
genStagingFilename(srvObj.getCfg(),
tmpReqPropsObj,
trgDiskInfo, fio.getFileId())
# Receive the data into the Staging File using the urllib.
if (srvObj.getHostId() != hostId):
# Example: http://host:7777/RETRIEVE?file_id=id&file_version=1
ipAddress = hostInfoDic[hostId].getIpAddress()
portNo = hostInfoDic[hostId].getSrvPort()
fileUrl = "http://" + ipAddress + ":" + str(portNo) +\
"/RETRIEVE?" + "file_id=" + fio.getFileId() +\
"&file_version=" + str(fio.getFileVersion())
# If a specific Disk ID for the source file is given, append
# this.
if (fio.getDiskId()):
fileUrl += "&disk_id=%s" % fio.getDiskId()
# Check if host is suspended, if yes, wake it up.
if (srvObj.getDb().getSrvSuspended(hostId)):
logger.debug("Clone Request - Waking up suspended " +\
"NGAS Host: %s", hostId)
ngamsSrvUtils.wakeUpHost(srvObj, hostId)
else:
fileUrl = "file:" + mtPt + "/" + fio.getFilename()
logger.debug(
"Receiving file via URI: %s into staging filename: %s",
fileUrl, stagingFilename)
# We try up to 5 times to retrieve the file in case a problem is
# encountered during cloning.
for attempt in range(5):
try:
filename, headers = urllib.urlretrieve(
fileUrl, stagingFilename)
_checkFile(srvObj, fio, stagingFilename, headers,
checkChecksum)
# If we get to this point the transfer was (probably) OK.
break
except Exception, e:
rmFile(stagingFilename)
errMsg = "Problem occurred while cloning file "+\
"via URL: " + fileUrl + " - Error: " + str(e)
if (attempt < 4):
errMsg += " - Retrying in 5s ..."
logger.error(errMsg)
time.sleep(5)
else:
raise Exception, errMsg
# We simply copy the file into the same destination as the
# source file (but on another disk).
targPathName = os.path.dirname(fio.getFilename())
targFilename = os.path.basename(fio.getFilename())
complTargPath = os.path.normpath(trgDiskInfo.getMountPoint() +\
"/" + targPathName)
checkCreatePath(complTargPath)
complFilename = os.path.normpath(complTargPath + "/" +
targFilename)
mvTime = mvFile(stagingFilename, complFilename)
ngamsLib.makeFileReadOnly(complFilename)
# Update status for new file in the DB.
newFileInfo = fio.clone().setDiskId(trgDiskInfo.getDiskId()).\
setCreationDate(getFileCreationTime(complFilename))
fileExists = srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(),
fio.getFileId(),
fio.getFileVersion())
newFileInfo.write(srvObj.getHostId(), srvObj.getDb())
# Update status for the Target Disk in DB + check if the disk is
# completed.
if (fileExists): mvTime = 0
dummyDapiStatObj = ngamsDapiStatus.ngamsDapiStatus().\
setDiskId(trgDiskInfo.getDiskId()).\
setFileExists(fileExists).\
setFileSize(fio.getFileSize()).setIoTime(mvTime)
ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), dummyDapiStatObj)
ngamsArchiveUtils.checkDiskSpace(srvObj, trgDiskInfo.getDiskId())
# Update the clone file status list.
if (emailNotif):
tmpFileList.setStatus(NGAMS_SUCCESS)
tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE"))
tmpFileList.addFileInfoObj(newFileInfo.setTag("TARGET_FILE"))
cloneStatusDbm.addIncKey(tmpFileList)
successCloneCount += 1
# If running as a cache archive, update the Cache New Files DBM
# with the information about the new file.
if (srvObj.getCachingActive()):
diskId = trgDiskInfo.getDiskId()
fileId = fio.getFileId()
fileVer = fio.getFileVersion()
filename = fio.getFilename()
ngamsCacheControlThread.addEntryNewFilesDbm(
srvObj, diskId, fileId, fileVer, filename)
# Generate a confirmation log entry.
cloneTime = time.time() - clone_start
timeAccu += cloneTime
msg = genLog("NGAMS_INFO_FILE_CLONED", [
fio.getFileId(),
fio.getFileVersion(),
fio.getDiskId(), hostId
])
msg = msg + ". Time: %.3fs. Total time: %.3fs." %\
(cloneTime, timeAccu)
logger.info(msg, extra={'to_syslog': True})
except Exception, e:
cloneTime = time.time() - clone_start
timeAccu += cloneTime
errMsg = genLog("NGAMS_ER_FILE_CLONE_FAILED", [
fio.getFileId(),
fio.getFileVersion(),
fio.getDiskId(), hostId,
str(e)
])
if (abortCloneLoop):
logger.error(errMsg, extra={'to_syslog': True})
thread.exit()
else:
logger.warning(errMsg)
if (emailNotif):
tmpFileList.setStatus(NGAMS_FAILURE + ": Error: " + errMsg)
tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE"))
cloneStatusDbm.addIncKey(tmpFileList)
failedCloneCount += 1
# Delete Staging File if already created.
if ((stagingFilename != "") and (os.path.exists(stagingFilename))):
rmFile(stagingFilename)
def cloneCheckDiskSpace(srvObj, cloneListDbmName, tmpFilePat, targetDiskId=""):
"""
Check if there is enough disk space available on this NGAS host for
carrying out the Clone Request.
srvObj: Reference to instance of Server Object (ngamsServer).
cloneListDbmName: Name of DBM containing the information about
the files to be cloned. This DB has an index number
as key pointing to pickled information about each file.
This pickled information is
[<File Info Object>, <Host ID>, <Mount Point>]
(string)
tmpFilePat: File pattern to be used for generating temporary
files (string).
targetDiskId: ID of disk to where the files cloned should be
written (string).
Returns: Void.
"""
T = TRACE()
# Make a dictionary with the available amount of space per disk.
logger.debug("Generating dictionary with available space per disk")
availDiskSpaceDic = {}
mtRootDir = srvObj.getCfg().getRootDirectory()
if (targetDiskId):
tmpDiskInfoObjList = [ngamsDiskInfo.ngamsDiskInfo().\
read(srvObj.getDb(), targetDiskId)]
else:
tmpDiskInfoObjList = ngamsDiskUtils.\
getDiskInfoForMountedDisks(srvObj.getDb(),
srvObj.getHostId(),
mtRootDir)
for diskInfoObj in tmpDiskInfoObjList:
mtPt = diskInfoObj.getMountPoint()
diskId = diskInfoObj.getDiskId()
availDiskSpaceDic[diskId] = getDiskSpaceAvail(mtPt, format="B")
# Now simulate the execution of the clone job by going sequentially
# through the files selected for cloning and subtract the required
# disk space from the available amount for each disk.
#
# When a disk reaches the threshold for considering the disk as
# completed, it is removed from the list of available disks.
cloneListDbm = ngamsDbm.ngamsDbm(cloneListDbmName)
# This dictionary contains a list of files that have been simulated
# successfully cloned. Since they are not updated in the DB, we keep
# track of this locally.
cloneSucDbmName = tmpFilePat + "_CLONE_SUCCESS_DB"
rmFile(cloneSucDbmName + "*")
cloneSucDbm = ngamsDbm.ngamsDbm(cloneSucDbmName,
cleanUpOnDestr=1,
writePerm=1)
# This is used to keep track of the files that could not be cloned
# due to lack of space.
cloneFailDbmName = tmpFilePat + "_CLONE_FAILED_DB"
rmFile(cloneFailDbmName + "*")
cloneFailDbm = ngamsDbm.ngamsDbm(cloneFailDbmName,
cleanUpOnDestr=1,
writePerm=1)
# This is used to simulate disk completion. If a specific target
# disk is defined, all other disks should be ignored (=we set them to
# completed).
cloneSimDiskCompl = []
if (targetDiskId):
tmpDiskList = ngamsDiskUtils.\
getDiskInfoForMountedDisks(srvObj.getDb(), srvObj.getHostId(),
mtRootDir)
for idx in range(len(tmpDiskList)):
if (tmpDiskList[idx].getDiskId() != targetDiskId):
cloneSimDiskCompl.append(tmpDiskList[idx].getDiskId())
# Carry out the simulated clone process.
ngamsDiskUtils.findTargetDiskResetCache()
key = 0
while (1):
if (not cloneListDbm.hasKey(str(key))): break
fileInfo = cloneListDbm.get(str(key))
key += 1
fio = fileInfo[0]
hostName = fileInfo[1]
text = "Simulating cloning of file - File ID: %s/%d, on disk " +\
"with ID: %s on host: %s"
logger.debug(text, fio.getFileId(), fio.getFileVersion(),
fio.getDiskId(), hostName)
diskExemptList = cloneSimDiskCompl + [fio.getDiskId()]
trgDiskInfo = None
while (1):
try:
trgDiskInfo = ngamsDiskUtils.\
findTargetDisk(srvObj.getHostId(),
srvObj.getDb(), srvObj.getCfg(),
fio.getFormat(), 0,
diskExemptList, 1)
except Exception, e:
if (str(e).find("NGAMS_AL_NO_STO_SETS") != -1):
# No more candidate Target Disks for this type
# of data - this file cannot be cloned.
cloneFailDbm.addIncKey(fio)
break
# Check if a file with that ID + version is already
# stored on the selected Target Disk.
fileInDb = srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(),
fio.getFileId(),
fio.getFileVersion())
fileKey = ngamsLib.genFileKey(trgDiskInfo.getDiskId(),
fio.getFileId(),
fio.getFileVersion())
fileSimCloned = cloneSucDbm.hasKey(fileKey)
if (fileInDb or fileSimCloned):
# This file is already stored on the given disk.
# Add to the exempt list.
diskExemptList.append(trgDiskInfo.getDiskId())
else:
# OK, this disk should be OK, stop looking for a
# suitable Target Disk.
logger.debug("Found suitable disk with ID: %s/Slot ID: %s",
trgDiskInfo.getDiskId(), trgDiskInfo.getSlotId())
cloneSucDbm.add(fileKey, "")
break
# We now subtract the size of the file from the available amount of
# disk space for the selected Target Disk. When the amount of available
# space goes below the threshold defined for this NG/AMS system that
# disk is considered to be completed.
if (trgDiskInfo):
diskId = trgDiskInfo.getDiskId()
availDiskSpaceDic[diskId] -= float(fio.getFileSize())
if ((availDiskSpaceDic[diskId] / 1048576.0) < \
srvObj.getCfg().getFreeSpaceDiskChangeMb()):
cloneSimDiskCompl.append(diskId)
def handleCmd(srvObj, reqPropsObj, httpRef):
"""
Handle Command QUERY to query the DB system used.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
T = TRACE()
# Get command parameters.
query = None
args = ()
if (reqPropsObj.hasHttpPar("query")):
query = reqPropsObj.getHttpPar("query")
qkey = query
if query.lower() in valid_queries.keys():
query = valid_queries[query.lower()]
else:
msg = "Invalid query specified. Valid queries are: %s" %\
valid_queries.keys()
raise Exception, msg
if reqPropsObj.getHttpPar(
"query") == 'subscribers_like' or reqPropsObj.getHttpPar(
"query") == 'files_like' or reqPropsObj.getHttpPar(
"query") == 'files_location' or reqPropsObj.getHttpPar(
"query") == 'lastver_location':
param = '%'
if (reqPropsObj.hasHttpPar("like")):
param = reqPropsObj.getHttpPar("like")
args = (param, )
if reqPropsObj.getHttpPar("query") == 'files_between':
param1 = param2 = ''
if (reqPropsObj.hasHttpPar("start")):
param1 = reqPropsObj.getHttpPar("start")
if (reqPropsObj.hasHttpPar("end")):
param2 = reqPropsObj.getHttpPar("end")
if param1 and param2:
args = (param1, param2)
elif param1:
query = 'select * from ngas_files where ingestion_date >= {0}'
args = (param1, )
else:
query = valid_queries['files_list']
else:
msg = "No query specified. Valid queries are: %s" %\
valid_queries.keys()
raise Exception, msg
out_format = None
if (reqPropsObj.hasHttpPar("format")):
out_format = reqPropsObj.getHttpPar("format")
cursorId = None
if (reqPropsObj.hasHttpPar("cursor_id")):
cursorId = reqPropsObj.getHttpPar("cursor_id")
fetch = None
if (reqPropsObj.hasHttpPar("fetch")):
fetch = int(reqPropsObj.getHttpPar("fetch"))
# Execute the query.
if (not cursorId):
res = srvObj.getDb().query2(query, args=args)
# TODO: Make possible to return an XML document
# TODO: Potential problem with very large result sets.
# Implement streaming result directly.
if (out_format == "list"):
header = None
if reqPropsObj.getHttpPar("query") not in [
'files_stats', 'files_list_recent', 'files_location',
'lastver_location'
]:
if query.find('ngas_files') >= 0:
header = NGAMS_FILES_COLS
elif query.find('ngas_disks') >= 0:
header = NGAMS_DISKS_COLS
elif query.find('ngas_subscribers') >= 0:
header = NGAMS_SUBSCR_COLS
elif reqPropsObj.getHttpPar("query") == 'files_stats':
header = ['Number of files', 'Total volume [MB]']
elif reqPropsObj.getHttpPar("query") == 'files_list_recent':
header = [
'file_id', 'file_name', 'file_size', 'ingestion_date'
]
elif reqPropsObj.getHttpPar("query") == 'files_location':
header = [
'host_id', 'file_full_path', 'file_version',
'ingestion_date'
]
elif reqPropsObj.getHttpPar("query") == 'lastver_location':
header = [
'host_id', 'file_full_path', 'file_version',
'creation_date', 'ingestion_date'
]
finalRes = formatAsList(res, header=header)
"""
if query.find('ngas_files') >=0:
header = NGAMS_FILES_COLS
elif query.find('ngas_disks') >= 0:
header = NGAMS_DISKS_COLS
else:
header = None
finalRes = formatAsList(res, header=header)
"""
mimeType = NGAMS_TEXT_MT
elif (out_format == "pickle"):
finalRes = cPickle.dumps([res])
mimeType = NGAMS_PYTHON_PICKLE_MT
elif (out_format == "json"):
jsobj = createJsonObj(res, qkey)
finalRes = json.dumps(jsobj, default=encode_decimal)
mimeType = NGAMS_JSON_MT
else:
finalRes = str([res])
mimeType = NGAMS_PYTHON_LIST_MT
# Return the data.
httpRef.send_data(finalRes, mimeType)
elif (fetch):
cursorDbmFilename = genCursorDbmName(srvObj.getCfg().\
getRootDirectory(), cursorId)
if (not os.path.exists(cursorDbmFilename)):
logger.error("Illegal Cursor ID: %s or cursor expired", cursorId)
return []
try:
cursorDbm = ngamsDbm.ngamsDbm(cursorDbmFilename, writePerm=1)
count = 0
resSet = []
cursorIdx = cursorDbm.get(CURSOR_IDX)
while (count < fetch):
cursorIdx += 1
if (cursorDbm.hasKey(str(cursorIdx))):
res = cursorDbm.get(str(cursorIdx))
resSet.append(res)
else:
# -- no more results to return.
break
count += 1
cursorDbm.add(CURSOR_IDX, cursorIdx)
cursorDbm.sync()
del cursorDbm
# If all entries have been fetched, we delete the cursor DBM.
if (count < fetch):
rmFile(cursorDbmFilename + "*")
# Return the data.
# TODO: Make it possible to return ASCII List + XML.
httpRef.send_data(str(resSet), NGAMS_PYTHON_LIST_MT)
except Exception, e:
msg = "Error fetching from cursor with ID: %s. Error: %s"
raise Exception, msg % (cursorId, str(e))
def _remStatErrReport(srvObj,
reqPropsObj,
tmpFilePat,
filesMisCopiesDbmName,
filesNotRegDbmName,
fileListDbmName,
diskId=None,
fileId=None,
fileVersion=None):
"""
Function to generate an error report in case of the REMDISK and REMFILE
commands.
srvObj: Reference to NG/AMS server class object
(ngamsServer).
reqPropsObj: Request Property object to keep track of actions
done during the request handling (ngamsReqProps).
tmpFilePat: File pattern to be used when generating temporary
files (string).
filesMisCopiesDbmName: Name of DBM containing references to files with
less than the required number of copies (string).
filesNotRegDbmName: Name of DBM containing complete filenames as keys
referring to files, which are not registered in
the DB (string).
fileListDbmName: Name of DBM containing all information about the
files concerned with the query.
diskId: Disk ID if relevant (string).
fileId: File ID if relevant (string).
fileVersion: File Version if relevant (integer).
Returns: In case inconsistencies were found, an ngamsStatus
object with more information is returned. Otherwise,
if everything was found to be OK, None is returned
(ngamsStatus|None).
"""
cmd = reqPropsObj.getCmd()
hostId = srvObj.getHostId()
#########################################################################
# At least three copies of each file?
#########################################################################
filesMisCopyDbm = ngamsDbm.ngamsDbm(filesMisCopiesDbmName, writePerm=0)
misFileCopies = filesMisCopyDbm.getCount()
if (misFileCopies and reqPropsObj.hasHttpPar("notif_email")):
xmlReport = 0
if (xmlReport):
# TODO:
pass
else:
statRep = tmpFilePat + "_NOTIF_EMAIL.txt"
fo = open(statRep, "w")
tmpFormat = cmd + " STATUS REPORT - " +\
"MISSING FILE COPIES:\n\n" +\
"==Summary:\n\n" +\
"Date: %s\n" +\
"NGAS Host ID: %s\n" +\
"Disk ID: %s\n" +\
"Files Detected: %d\n\n" +\
"==File List:\n\n"
fo.write(tmpFormat % (toiso8601(), hostId, diskId, misFileCopies))
tmpFormat = "%-32s %-12s %-6s\n"
fo.write(tmpFormat % ("File ID", "Version", "Copies"))
fo.write(tmpFormat % (32 * "-", 7 * "-", 6 * "-"))
# Loop over the files an generate the report.
filesMisCopyDbm.initKeyPtr()
while (1):
fileKey, fileInfo = filesMisCopyDbm.getNext()
if (not fileKey): break
noOfCopies = fileInfo.getTag().split(": ")[-1]
fo.write(tmpFormat %
(fileInfo.getFileId(), str(
fileInfo.getFileVersion()), noOfCopies))
fo.write(52 * "-")
fo.write("\n\n==END\n")
fo.close()
mimeType = NGAMS_TEXT_MT
# Send out Notification Email.
_notify(srvObj, reqPropsObj, statRep, mimeType, cmd)
del filesMisCopyDbm
# Generate error message if files with less than the required number
# of copies are concerned by the query.
if (misFileCopies):
errMsg = genLog("NGAMS_WA_FILE_COPIES")
status = srvObj.genStatus(NGAMS_FAILURE, errMsg)
status.setMessage(status.getMessage() +\
" Cannot remove item: %s/%s/%s" %
(str(diskId), str(fileId), str(fileVersion)))
logger.warning(errMsg)
return status
#########################################################################
#########################################################################
# Check for spurious files.
#########################################################################
spuriousFilesDbmName = checkSpuriousFiles(srvObj, tmpFilePat, hostId,
diskId, fileId, fileVersion)
spuriousFilesDbm = ngamsDbm.ngamsDbm(spuriousFilesDbmName, writePerm=0)
spuriousFiles = spuriousFilesDbm.getCount()
srvDataChecking = srvObj.getDb().getSrvDataChecking(hostId)
if (spuriousFiles and reqPropsObj.hasHttpPar("notif_email")):
xmlReport = 0
if (xmlReport):
# TODO:
pass
else:
statRep = tmpFilePat + "_NOTIF_EMAIL.txt"
fo = open(statRep, "w")
tmpFormat = cmd + " STATUS REPORT - SPURIOUS FILES:\n\n" +\
"==Summary:\n\n" +\
"Date: %s\n" +\
"NGAS Host ID: %s\n" +\
"Disk ID: %s\n" +\
"Spurious Files: %d\n"
fo.write(tmpFormat % (toiso8601(), hostId, diskId, spuriousFiles))
if (srvDataChecking):
fo.write("Note: NGAS Host is performing Data Consistency " +\
"Checking - consider to switch off!\n")
fo.write("\n==File List:\n\n")
tmpFormat = "%-32s %-12s %-11s %-11s\n"
fo.write(tmpFormat %
("File ID", "File Version", "File Status", "Ignore Flag"))
fo.write(tmpFormat % (32 * "-", 12 * "-", 11 * "-", 11 * "-"))
# Loop over the files an generate the report.
spuriousFilesDbm.initKeyPtr()
while (1):
key, fileInfo = spuriousFilesDbm.getNext()
if (not key): break
fileId = fileInfo[ngamsDbCore.SUM1_FILE_ID]
fileVersion = str(fileInfo[ngamsDbCore.SUM1_VERSION])
fileStatus = fileInfo[ngamsDbCore.SUM1_FILE_STATUS]
ignoreFlag = str(fileInfo[ngamsDbCore.SUM1_FILE_IGNORE])
fo.write(tmpFormat %
(fileId, fileVersion, fileStatus, ignoreFlag))
fo.write("\n\n==END\n")
fo.close()
mimeType = NGAMS_TEXT_MT
# Send out Notification Email.
_notify(srvObj, reqPropsObj, statRep, mimeType, cmd)
del spuriousFilesDbm
# Generate error message if spurious files were found.
if (spuriousFiles):
errMsg = genLog("NGAMS_WA_FILE_NON_REM")
logger.warning(errMsg)
status = srvObj.genStatus(NGAMS_FAILURE, errMsg)
msg = status.getMessage() + " Cannot delete specified files"
if (srvDataChecking):
msg += ". Note: NGAS Host is performing Data Consistency " +\
"Checking - consider to switch off!"
status.setMessage(msg)
return status
#########################################################################
#########################################################################
# Unregistered files found?
#########################################################################
filesNotRegDbm = ngamsDbm.ngamsDbm(filesNotRegDbmName, writePerm=0)
unRegFilesFound = filesNotRegDbm.getCount()
if (unRegFilesFound and reqPropsObj.hasHttpPar("notif_email")):
xmlReport = 0
if (xmlReport):
# TODO:
pass
else:
statRep = tmpFilePat + "_NOTIF_EMAIL.txt"
fo = open(statRep, "w")
tmpFormat = cmd + " STATUS REPORT - " +\
"NON-REGISTERED FILES:\n\n" +\
"==Summary:\n\n" +\
"Date: %s\n" +\
"NGAS Host ID: %s\n" +\
"Disk ID: %s\n" +\
"Non-Registered Files: %d\n\n" +\
"==File List:\n\n"
fo.write(tmpFormat %
(toiso8601(), hostId, diskId, unRegFilesFound))
# Loop over the files an generate the report.
filesNotRegDbm.initKeyPtr()
while (1):
fileName, dummy = filesNotRegDbm.getNext()
if (not fileName): break
fo.write(fileName + "\n")
fo.write("\n\n==END\n")
fo.close()
mimeType = NGAMS_TEXT_MT
# Send out Notification Email.
_notify(srvObj, reqPropsObj, statRep, mimeType, cmd)
del filesNotRegDbm
# Generate error message if non-registered files found.
if (unRegFilesFound):
errMsg = genLog("NGAMS_WA_FILES_NOT_REG")
logger.warning(errMsg)
status = srvObj.genStatus(NGAMS_FAILURE, errMsg)
status.setMessage(status.getMessage() +\
" Cannot remove volume: %s/%s/%s" %
(str(diskId), str(fileId), str(fileVersion)))
return status
#########################################################################
#########################################################################
# Check if the files to be removed actually can be removed.
#########################################################################
nonRemFilesDbmName = checkFilesRemovable(srvObj, fileListDbmName,
tmpFilePat)
nonRemFilesDbm = ngamsDbm.ngamsDbm(nonRemFilesDbmName, writePerm=0)
nonRemFiles = nonRemFilesDbm.getCount()
if (nonRemFiles and reqPropsObj.hasHttpPar("notif_email")):
xmlReport = 0
if (xmlReport):
# TODO:
pass
else:
statRep = tmpFilePat + "_NOTIF_EMAIL.txt"
fo = open(statRep, "w")
tmpFormat = cmd + " STATUS REPORT - " +\
"NON-REMOVABLE FILES:\n\n" +\
"==Summary:\n\n" +\
"Date: %s\n" +\
"NGAS Host ID: %s\n" +\
"Disk ID: %s\n" +\
"Non-Removable Files: %d\n\n" +\
"==File List:\n\n"
fo.write(tmpFormat % (toiso8601(), hostId, diskId, nonRemFiles))
tmpFormat = "%-32s %-12s %-32s\n"
fo.write(tmpFormat % ("File ID", "File Version", "Filename"))
fo.write(tmpFormat % (32 * "-", 12 * "-", 32 * "-"))
# Loop over the files an generate the report.
nonRemFilesDbm.initKeyPtr()
while (1):
key, fileInfo = nonRemFilesDbm.getNext()
if (not key): break
fo.write(tmpFormat % (fileInfo[ngamsDbCore.SUM1_FILE_ID],
fileInfo[ngamsDbCore.SUM1_VERSION],
fileInfo[ngamsDbCore.SUM1_FILENAME]))
fo.write("\n\n==END\n")
fo.close()
mimeType = NGAMS_TEXT_MT
# Send out Notification Email.
_notify(srvObj, reqPropsObj, statRep, mimeType, cmd)
del nonRemFilesDbm
# Generate error message if files concerned by the query could not be
# deleted.
if (nonRemFiles):
errMsg = genLog("NGAMS_WA_FILE_NON_REM")
logger.warning(errMsg)
status = srvObj.genStatus(NGAMS_FAILURE, errMsg)
status.setMessage(status.getMessage() +\
" Cannot delete specified files")
return status
#########################################################################
# If we got to this point, no problems where found.
return None
def _remDisk(srvObj, reqPropsObj, diskId, execute, tmpFilePat):
"""
See documentation for the ngamsRemDiskCmd.remDisk() function.
"""
# Get disk info -- if any available.
sqlDiskInfo = srvObj.getDb().getDiskInfoFromDiskId(diskId)
diskInfo = ngamsDiskInfo.ngamsDiskInfo()
if (sqlDiskInfo != []):
if (diskInfo.unpackSqlResult(sqlDiskInfo).getHostId() !=
srvObj.getHostId()):
sqlDiskInfo = None
# Check that the disk is mounted in this unit (no proxy for the REMDISK
# Command ...).
if (diskInfo.getHostId() != srvObj.getHostId()):
errMsg = "Disk referred to by Disk ID: %s seems not to be mounted " +\
"in this unit: %s -- rejecting REMDISK Command"
errMsg = errMsg % (diskId, srvObj.getHostId())
raise Exception(errMsg)
# Check that execution of the request can be granted.
filesMisCopyDbmName, filesNotRegDbmName, fileListDbmName =\
ngamsRemUtils.checkFileCopiesAndReg(srvObj, 3,
tmpFilePat, None,
diskId)
status = ngamsRemUtils._remStatErrReport(srvObj, reqPropsObj, tmpFilePat,
filesMisCopyDbmName,
filesNotRegDbmName,
fileListDbmName, diskId)
misCopiesDbm = ngamsDbm.ngamsDbm(filesMisCopyDbmName)
if (misCopiesDbm.getCount() == 0):
logger.info("Disk with ID: %s approved for deletion", diskId)
else:
logger.info("Disk with ID: %s rejected for deletion", diskId)
if (status): return status
# Check if there is enough space on the disk to store the file info
# cache files.
kbPerFile = 0.256
tmpDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm=0)
noOfFiles = tmpDbm.getCount()
del tmpDbm
kbsAvail = getDiskSpaceAvail(diskInfo.getMountPoint(), format="KB")
kbsReq = (kbPerFile * noOfFiles)
msg = "Space required for REMDISK Command: %.1f KB, " +\
"space available: %.1f KB"
logger.debug(msg, kbsReq, kbsAvail)
if (kbsReq > kbsAvail):
errMsg = "Not enough space on disk to carry out REMDISK Command. " +\
"Host: %s, Disk ID: %s, Mount Point: %s. " +\
"Required disk space: %.3f MB"
errMsg = errMsg % (getHostName(), diskInfo.getDiskId(),
diskInfo.getMountPoint(), (kbsReq / 1024.))
raise Exception(errMsg)
#########################################################################
# Execute the deletion if execute = 1 and a matching disk was found.
# Otherwise just return the ngamsStatus object as confirmation.
#########################################################################
if (execute and (sqlDiskInfo != [])):
# We delete first the info in the DB in connection with the disk and
# afterwards the contents on the disk. This is done like this, since
# it is considered worse having entries for files in the DB which are
# not available on the disk since NGAS uses the info in the DB to check
# for the number of available copies.
logger.info("Removing DB record for disk with ID: %s", diskId)
try:
tmpDir = os.path.dirname(tmpFilePat)
srvObj.getDb().deleteDiskInfo(diskId, 1)
except Exception as e:
errMsg = genLog("NGAMS_ER_DEL_DISK_DB", [diskId, str(e)])
raise Exception(errMsg)
logger.info("Deleting contents on disk with ID: %s", diskId)
try:
rmFile(os.path.normpath(diskInfo.getMountPoint() + "/*"))
except Exception as e:
errMsg = genLog("NGAMS_ER_DEL_DISK", [diskId, str(e)])
raise Exception(errMsg)
try:
# Remember to remove entry for disk from the Physical Disk Dic.
del srvObj.getDiskDic()[diskInfo.getSlotId()]
except:
pass
infoMsg = genLog("NGAMS_INFO_DEL_DISK", [diskId])
logger.info(infoMsg)
# Add entry in the NGAS Disks History Table.
ngasDiskInfo = ngamsDiskUtils.prepNgasDiskInfoFile(
srvObj.getHostId(), diskInfo, 1, 1)
srvObj.getDb().addDiskHistEntry(srvObj.getHostId(), diskId,
"Disk Removed", NGAMS_XML_MT,
ngasDiskInfo)
elif (sqlDiskInfo != []):
infoMsg = genLog("NGAMS_INFO_DEL_DISK_SEL", [diskId])
else:
infoMsg = genLog("NGAMS_WA_DEL_DISK2", [srvObj.getHostId(), diskId])
# Generate status.
status = srvObj.genStatus(NGAMS_SUCCESS, infoMsg)
if (sqlDiskInfo): status.addDiskStatus(diskInfo)
#########################################################################
return status
def _registerExec(srvObj,
fileListDbmName,
tmpFilePat,
diskInfoDic,
reqPropsObj=None):
"""
Register the files listed in the File List DBM (ngamsDbm), which
match the mime-type(s) either specified in the 'mimeType' parameter,
or if this is not specified, which match all the mime-types specified
in the configuration file.
When the registration procedure has been executed, the function sends
an Email Notification message indicating which files were registered
if the HTTP parameter 'notif_email' is given.
The functions creates a File Info Objects per file handled and
writes this in a temporary file, which is a DBM file. The
keys in this DB is simply the file number in the sequence of files
handled, pointing to a pickled ngamsFileInfo object.
Each of the File Info Objects indicates if the file was registered or not.
This is done by setting the tag of the File Info Object to one of the
following values:
REGISTERED: The file was successfully registered in the NGAS DB
FAILED: The file was selected for registration but could not
be properly registered because of inconsistencies.
The status will be of the format: 'FAILED[: <reason>]'.
REJECTED: A file found under the specified path directory was
not accepted for cloning, usually because the mime-type
was not correct. The status will be of the format:
'REJECTED[: <reason>]'.
Note, that registration is comparable to archiving of files. For that
reason a DAPI must be provided for each type of file that should be
registered. If this is not fullfilled, the file registration will
fail.
Only files stored on one of the NGAS disks configured in the
configuration file, are considered. Files stored in other locations
are ignored.
srvObj: Instance of NG/AMS Server object (ngamsServer).
fileListDbmName: Name of a DBM containing the information
about the files to be registered. Each element in the
list is referred to by a key, which is a number.
These points to a pickled list for each file containing
the following information:
[<Filename>, <Disk ID>, <Mime-Type>]
The information for each disk concerned is also contained
in the DB referred to by its Disk ID and by its mount
point. The data is a pickled instance of the ngamsDiskInfo
class (string).
tmpFilePat: Pattern for temporary files used during the registration
process (string).
diskInfoDic: Dictionary with Disk IDs as keys pointing to the info
about the disk (dictionary/ngamsDiskInfo).
reqPropsObj: If an NG/AMS Request Properties Object is given, the
Request Status will be updated as the request is carried
out (ngamsReqProps).
Returns: Void.
"""
emailNotif = 0
if (reqPropsObj):
if (reqPropsObj.hasHttpPar("notif_email")):
emailNotif = 1
# Create the temporary BSD DB to contain the information for the
# Email Notification Message.
if (emailNotif):
regDbmName = tmpFilePat + "_NOTIF_EMAIL"
regDbm = ngamsDbm.ngamsDbm(regDbmName, writePerm=1)
# Open the DBM containing the list of files to (possibly) register.
fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm=1)
# Want to parse files in alphabetical order.
# TODO: Portatibility issue. Try to avoid UNIX shell commands for sorting.
tmpFileList = tmpFilePat + "_FILE_LIST"
rmFile(tmpFileList)
with open(tmpFileList, "wb") as fo:
fileListDbm.initKeyPtr()
while (1):
dbmKey, fileInfo = fileListDbm.getNext()
if (not dbmKey): break
fo.write(dbmKey + b"\n")
sortFileList = tmpFilePat + "_SORT_FILE_LIST"
rmFile(sortFileList)
shellCmd = "sort %s > %s" % (tmpFileList, sortFileList)
stat, out, err = ngamsCore.execCmd(shellCmd)
if (stat != 0):
raise Exception("Error executing command: %s. Error: %s, %s" %\
(shellCmd, str(out), str(err)))
rmFile(tmpFileList)
# Go through each file in the list, check if the mime-type is among the
# ones, which apply for registration. If yes try to register the file
# by invoking the corresponding DAPI on the file.
fileRegCount = 0
fileFailCount = 0
fileRejectCount = 0
regTimeAccu = 0.0
fileCount = 0
fo = open(sortFileList)
run = 1
while (run):
reg_start = time.time()
dbmKey = fo.readline()
if (dbmKey.strip() == ""):
run = 0
continue
fileInfo = fileListDbm.get(dbmKey[0:-1])
filename = fileInfo[0]
diskId = fileInfo[1]
mimeType = fileInfo[2]
# Register the file. Check first, that exactly this file is
# not already registered. In case it is, the file will be rejected.
regPi = srvObj.getCfg().register_plugins[mimeType]
logger.debug("Plugin found for %s: %s", mimeType, regPi)
params = ngamsPlugInApi.parseRawPlugInPars(regPi.pars)
tmpReqPropsObj = ngamsReqProps.ngamsReqProps().\
setMimeType(mimeType).\
setStagingFilename(filename).\
setTargDiskInfo(diskInfoDic[diskId]).\
setHttpMethod(NGAMS_HTTP_GET).\
setCmd(NGAMS_REGISTER_CMD).\
setSize(os.path.getsize(filename)).\
setFileUri(filename).\
setNoReplication(1)
tmpFileObj = ngamsFileInfo.ngamsFileInfo()
try:
# Invoke Registration Plug-In.
piName = regPi.name
plugInMethod = loadPlugInEntryPoint(piName)
piRes = plugInMethod(srvObj, tmpReqPropsObj, params)
del tmpReqPropsObj
# Check if this file is already registered on this disk. In case
# yes, it is not registered again.
files = srvObj.db.getFileSummary1(srvObj.getHostId(),
[piRes.getDiskId()],
[piRes.getFileId()])
fileRegistered = 0
for tmpFileInfo in files:
tmpMtPt = tmpFileInfo[ngamsDbCore.SUM1_MT_PT]
tmpFilename = tmpFileInfo[ngamsDbCore.SUM1_FILENAME]
tmpComplFilename = os.path.normpath(tmpMtPt + "/" +\
tmpFilename)
if (tmpComplFilename == filename):
fileRegistered = 1
break
if (fileRegistered):
fileRejectCount += 1
tmpMsgForm = "REJECTED: File with File ID/Version: %s/%d " +\
"and path: %s is already registered on disk " +\
"with Disk ID: %s"
tmpMsg = tmpMsgForm % (piRes.getFileId(),
piRes.getFileVersion(), filename,
piRes.getDiskId())
logger.warning(tmpMsg + ". File is not registered again.")
if (emailNotif):
tmpFileObj.\
setDiskId(diskId).setFilename(filename).\
setTag(tmpMsg)
regDbm.addIncKey(tmpFileObj)
if (reqPropsObj):
reqPropsObj.incActualCount(1)
ngamsHighLevelLib.stdReqTimeStatUpdate(
srvObj, reqPropsObj, regTimeAccu)
regTimeAccu += time.time() - reg_start
fileCount += 1
continue
# Calculate checksum. We maintain the old name for backwards
# compatibility
crc_variant = srvObj.cfg.getCRCVariant()
if crc_variant == ngamsFileUtils.CHECKSUM_CRC32_INCONSISTENT:
crc_variant = 'ngamsGenCrc32'
checksum = ngamsFileUtils.get_checksum(65536, filename,
crc_variant) or ''
# Move file and update information about file in the NGAS DB.
mvFile(filename, piRes.getCompleteFilename())
ngamsArchiveUtils.updateFileInfoDb(srvObj, piRes, checksum,
crc_variant)
ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), piRes)
ngamsLib.makeFileReadOnly(piRes.getCompleteFilename())
if (emailNotif):
uncomprSize = piRes.getUncomprSize()
ingestDate = time.time()
creDateSecs = getFileCreationTime(filename)
tmpFileObj.\
setDiskId(diskId).\
setFilename(filename).\
setFileId(piRes.getFileId()).\
setFileVersion(piRes.getFileVersion()).\
setFormat(piRes.getFormat()).\
setFileSize(piRes.getFileSize()).\
setUncompressedFileSize(uncomprSize).\
setCompression(piRes.getCompression()).\
setIngestionDate(ingestDate).\
setIgnore(0).\
setChecksum(checksum).\
setChecksumPlugIn(crc_variant).\
setFileStatus(NGAMS_FILE_STATUS_OK).\
setCreationDate(creDateSecs).\
setTag("REGISTERED")
fileRegCount += 1
# If running as a cache archive, update the Cache New Files DBM
# with the information about the new file.
if (srvObj.getCachingActive()):
fileVer = fio.getFileVersion()
ngamsCacheControlThread.addEntryNewFilesDbm(
srvObj, diskId, piRes.getFileId(), fileVer, filename)
# Generate a confirmation log entry.
msg = genLog("NGAMS_INFO_FILE_REGISTERED", [
filename,
piRes.getFileId(),
piRes.getFileVersion(),
piRes.getFormat()
])
time.sleep(0.005)
regTime = time.time() - reg_start
msg = msg + ". Time: %.3fs." % (regTime)
logger.info(msg, extra={'to_syslog': 1})
except Exception as e:
errMsg = genLog("NGAMS_ER_FILE_REG_FAILED", [filename, str(e)])
logger.error(errMsg)
if (emailNotif):
tmpFileObj.\
setDiskId(diskId).setFilename(filename).\
setTag(errMsg)
fileFailCount += 1
regTime = time.time() - reg_start
# TODO (rtobar, 2016-01): Why don't we raise an exception here?
# Otherwise the command appears as successful on the
# client-side
# Add the file information in the registration report.
if (emailNotif): regDbm.addIncKey(tmpFileObj)
# Update request status time information.
regTimeAccu += regTime
if (reqPropsObj):
reqPropsObj.incActualCount(1)
ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj,
regTimeAccu)
fileCount += 1
fo.close()
rmFile(sortFileList)
if (emailNotif): regDbm.sync()
del fileListDbm
rmFile(fileListDbmName + "*")
# Final update of the Request Status.
if (reqPropsObj):
if (reqPropsObj.getExpectedCount() and reqPropsObj.getActualCount()):
complPercent = (100.0 * (float(reqPropsObj.getActualCount()) /
float(reqPropsObj.getExpectedCount())))
else:
complPercent = 100.0
reqPropsObj.setCompletionPercent(complPercent, 1)
reqPropsObj.setCompletionTime(1)
srvObj.updateRequestDb(reqPropsObj)
# Send Register Report with list of files cloned to a possible
# requestor(select) of this.
if (emailNotif):
xmlStat = 0
if (xmlStat):
# Create an instance of the File List Class, used to store the
# Registration Report.
regStat = ngamsFileList.\
ngamsFileList("FILE_REGISTRATION_STATUS",
"Status report for file " +\
"registration")
# Loop over the file objects in the BSD DB and add these
# in the status object.
regDbm.initKeyPtr()
while (1):
key, tmpFileObj = regDbm.getNext()
if (not key): break
regStat.addFileInfoObj(tmpFileObj)
# Set overall status of registration procedure.
regStat.setStatus("Files Found: " + str(fileCount + 1) + ", "+\
"Files Registered: " + str(fileRegCount) +\
", " +\
"Files Failed: " + str(fileFailCount) + ", " +\
"Files Rejected: " + str(fileRejectCount))
status = srvObj.genStatus(NGAMS_SUCCESS,
"REGISTER command status report").\
addFileList(regStat)
statRep = status.genXmlDoc()
statRep = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, statRep)
mimeType = NGAMS_XML_MT
else:
# Generate a 'simple' ASCII report.
statRep = tmpFilePat + "_NOTIF_EMAIL.txt"
fo = open(statRep, "w")
if (reqPropsObj):
path = reqPropsObj.getHttpPar("path")
else:
path = "-----"
if (fileCount):
timePerFile = (regTimeAccu / fileCount)
else:
timePerFile = 0
tmpFormat = "REGISTER STATUS REPORT:\n\n" +\
"==Summary:\n\n" +\
"Date: %s\n" +\
"NGAS Host: %s\n" +\
"Search Path: %s\n" +\
"Total Number of Files: %d\n" +\
"Number of Registered Files: %d\n" +\
"Number of Failed Files: %d\n" +\
"Number of Rejected Files: %d\n" +\
"Total processing time (s): %.3f\n" +\
"Handling time per file (s): %.3f\n\n" +\
"==File List:\n\n"
fo.write(tmpFormat % (toiso8601(), srvObj.getHostId(), path,
fileCount, fileRegCount, fileFailCount,
fileRejectCount, regTimeAccu, timePerFile))
tmpFormat = "%-80s %-32s %-3s %-10s\n"
fo.write(tmpFormat %\
("Filename", "File ID", "Ver", "Status"))
fo.write(tmpFormat % (80 * "-", 32 * "-", 3 * "-", 10 * "-"))
regDbm.initKeyPtr()
while (1):
key, tmpFileObj = regDbm.getNext()
if (not key): break
mtPt = diskInfoDic[tmpFileObj.getDiskId()].getMountPoint()
filename = os.path.normpath(mtPt + "/" +\
tmpFileObj.getFilename())
line = tmpFormat %\
(filename, tmpFileObj.getFileId(),
str(tmpFileObj.getFileVersion()),
tmpFileObj.getTag())
fo.write(line)
fo.write(128 * "-")
fo.write("\n\n==END\n")
fo.close()
mimeType = NGAMS_TEXT_MT
# Send out the status report.
emailAdrList = reqPropsObj.getHttpPar("notif_email").split(",")
attachmentName = "RegisterStatusReport"
if (reqPropsObj.hasHttpPar("path")):
attachmentName += "-" + reqPropsObj.getHttpPar("path").\
replace("/", "_")
ngamsNotification.notify(srvObj.host_id,
srvObj.cfg,
NGAMS_NOTIF_INFO,
"REGISTER STATUS REPORT",
statRep,
recList=emailAdrList,
force=1,
contentType=mimeType,
attachmentName=attachmentName)
del regDbm
rmFile(regDbmName + "*")
rmFile(statRep)
# Generate final status log + exit.
if (fileCount > 0):
timePerFile = (regTimeAccu / fileCount)
else:
timePerFile = 0.0
msg = "Registration procedure finished processing Register Request - " + \
"terminating. Files handled: %d. Total time: %.3fs. " + \
"Average time per file: %.3fs."
logger.debug(msg, fileCount, regTimeAccu, timePerFile)
def register(srvObj, path, mimeType="", reqPropsObj=None, httpRef=None):
"""
Function to generate a list of candidate files to register, and to
launch a thread that actually carries out the registration.
When the possible candidate files haven been selected, a reply is sent
back to the requestor if the 'httpRef' object is given.
srvObj: Instance of NG/AMS Server object (ngamsServer).
path: The path name, which is used as starting point for
the searching for files (string).
mimeType: Comma separated list of mime-types, which should be
considered for registration (string).
reqPropsObj: If an NG/AMS Request Properties Object is given, the
Request Status will be updated as the request is carried
out (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
# Check if the given path or file exists.
if (not os.path.exists(path)):
errMsg = genLog("NGAMS_ER_FILE_REG_FAILED",
[path, "Non-existing file or path."])
raise Exception(errMsg)
# Check if the given path/file is on one of the NGAS Disks.
foundPath = 0
for slotId in srvObj.getDiskDic().keys():
do = srvObj.getDiskDic()[slotId]
if (path.find(do.getMountPoint()) == 0):
foundPath = 1
break
if (not foundPath):
errMsg = genLog("NGAMS_ER_FILE_REG_FAILED", [
path, "File or path specified is not located on an "
"NGAS Disk."
])
raise Exception(errMsg)
# Create file pattern for temporary files.
tmpFilePat = ngamsHighLevelLib.genTmpFilename(srvObj.getCfg(),
"REGISTER_CMD")
# Generate dictionary with mime-types to accept.
mimeTypeDic = {}
if (mimeType != ""):
for mt in mimeType.split(","):
try:
mimeTypeDic[mt] = srvObj.getMimeTypeDic()[mt]
except:
errMsg = genLog("NGAMS_ER_REQ_HANDLING",
["Mime-type specified: " + mt + " " +\
"in connection with REGISTER command " +\
"does not have a DAPI defined"])
raise Exception(errMsg)
else:
mimeTypeDic = srvObj.getMimeTypeDic()
# From the Disk Dictionary, generate a dictionary with mappings from
# Disk ID + from Mount Point to the disk information for the disk.
diskInfoDic = {}
mtPt2DiskInfo = {}
for slotId in srvObj.getDiskDic().keys():
if (not srvObj.getCfg().getSlotIdDefined(slotId)): continue
diskId = srvObj.getDiskDic()[slotId].getDiskId()
mtPt = srvObj.getDiskDic()[slotId].getMountPoint()
tmpDiskInfo = ngamsDiskInfo.ngamsDiskInfo().\
read(srvObj.getDb(), diskId)
diskInfoDic[diskId] = tmpDiskInfo
mtPt2DiskInfo[mtPt] = tmpDiskInfo
# Generate a list with all files found under the specified path, which
# are candidates for being registered.
fileListDbmName = tmpFilePat + "_FILE_LIST"
rmFile(fileListDbmName + "*")
fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm=1)
mimeTypeMappings = srvObj.getCfg().getMimeTypeMappings()
tmpGlobFile = tmpFilePat + "_FILE_GLOB.glob"
fileCount = 0
searchPath = os.path.normpath(path)
foundVolume = False
for mtPt in mtPt2DiskInfo.keys():
mtPt2 = mtPt
if (mtPt[-1] != "/"): mtPt2 += "/"
if (searchPath.find(mtPt2) == 0):
foundVolume = True
if os.path.isdir(searchPath):
for root, dirs, files in os.walk(searchPath):
for nextFile in files:
if nextFile not in \
[NGAMS_DISK_INFO, \
NGAMS_VOLUME_ID_FILE, \
NGAMS_VOLUME_INFO_FILE]:
mimeType = ngamsLib.detMimeType(
mimeTypeMappings, nextFile, 0)
tmpFileInfo = [
os.path.join(root, nextFile),
mtPt2DiskInfo[mtPt].getDiskId(), mimeType
]
fileListDbm.add(os.path.join(root, nextFile),
tmpFileInfo)
elif os.path.isfile(
searchPath): # the path is actually pointing to a file
nextFile = os.path.basename(searchPath)
root = os.path.dirname(searchPath)
if nextFile not in \
[NGAMS_DISK_INFO, \
NGAMS_VOLUME_ID_FILE, \
NGAMS_VOLUME_INFO_FILE]:
mimeType = ngamsLib.detMimeType(mimeTypeMappings, nextFile,
0)
tmpFileInfo = [
os.path.join(root, nextFile),
mtPt2DiskInfo[mtPt].getDiskId(), mimeType
]
fileListDbm.add(os.path.join(root, nextFile), tmpFileInfo)
if foundVolume: break
# pattern = ""
# # TODO: Portatibility issue. The usage of UNIX commands should be
# # avoided if possible.
# while (1):
# # Use a shell commands here to avoid building up maybe huge lists in
# # memory/in the Python interpreter. A better way could maybe be found
# # avoiding to invoke a shell command.
# rmFile(tmpGlobFile)
# searchPath = os.path.normpath(path + pattern)
# tmpCmd = "find " + searchPath + " -maxdepth 1 > " + tmpGlobFile
# stat, out = commands.getstatusoutput(tmpCmd)
# if (stat != 0): break
# fo = open(tmpGlobFile)
# while (1):
# nextFile = fo.readline().strip()
# if (nextFile == ""): break
# if (nextFile.find("/" + NGAMS_DISK_INFO) != -1): continue
# if (nextFile.find("/" + NGAMS_VOLUME_ID_FILE) != -1): continue
# if (nextFile.find("/" + NGAMS_VOLUME_INFO_FILE) != -1): continue
# if (os.path.isfile(nextFile)):
# nextFile = os.path.normpath(nextFile)
#
# # Find the host disk. Only files located on mounted NGAS
# # disks are considered.
# for mtPt in mtPt2DiskInfo.keys():
# mtPt2 = mtPt
# if (mtPt[-1] != "/"): mtPt2 += "/"
# if (nextFile.find(mtPt2) == 0):
# info(4,"Found candidate file for registering: " +\
# nextFile)
# # Take only files with a Registration Plug-In defined.
# mimeType = ngamsLib.detMimeType(mimeTypeMappings,
# nextFile, 0)
# regPi = srvObj.getCfg().getRegPiFromMimeType(mimeType)
# if (regPi != None):
# tmpFileInfo = [nextFile,
# mtPt2DiskInfo[mtPt].getDiskId(),
# mimeType]
# fileListDbm.add(nextFile, tmpFileInfo)
# break
# fo.close()
# pattern += "/*"
# rmFile(tmpGlobFile)
fileListDbm.sync()
del fileListDbm
# Send intermediate reply if the HTTP Reference object is given.
is_async = 'async' in reqPropsObj and int(reqPropsObj['async'])
if httpRef and is_async:
logger.debug("REGISTER command accepted - generating immediate " +\
"confimation reply to REGISTER command")
# Update the request status in the Request Properties Object.
reqPropsObj.\
setExpectedCount(fileCount).\
setActualCount(0).setCompletionPercent(0)
srvObj.updateRequestDb(reqPropsObj)
status = srvObj.genStatus(NGAMS_SUCCESS,
"Accepted REGISTER command for execution").\
setReqStatFromReqPropsObj(reqPropsObj).\
setActualCount(0)
# Launch the register thread or run the command in foreground if wait=1
if is_async:
args = (srvObj, fileListDbmName, tmpFilePat, diskInfoDic, reqPropsObj,
None)
thrName = NGAMS_REGISTER_THR + threading.current_thread().getName()
regThread = threading.Thread(None, _registerThread, thrName, args)
regThread.setDaemon(0)
regThread.start()
else:
# Carry out the REGISTER Command (directly in this thread) and send
# reply when this is done.
_registerExec(srvObj, fileListDbmName, tmpFilePat, diskInfoDic,
reqPropsObj)
msg = "Successfully handled command REGISTER"
logger.debug(msg)
status = srvObj.genStatus(NGAMS_SUCCESS, msg).\
setReqStatFromReqPropsObj(reqPropsObj).setActualCount(0)
# Send reply if possible.
if (httpRef):
xmlStat = status.genXmlDoc(0, 0, 0, 1, 0)
xmlStat = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, xmlStat)
httpRef.send_data(six.b(xmlStat), NGAMS_XML_MT)
def _registerExec(srvObj,
fileListDbmName,
tmpFilePat,
diskInfoDic,
reqPropsObj = None):
"""
Register the files listed in the File List DBM (ngamsDbm), which
match the mime-type(s) either specified in the 'mimeType' parameter,
or if this is not specified, which match all the mime-types specified
in the configuration file.
When the registration procedure has been executed, the function sends
an Email Notification message indicating which files were registered
if the HTTP parameter 'notif_email' is given.
The functions creates a File Info Objects per file handled and
writes this in a temporary file, which is a DBM file. The
keys in this DB is simply the file number in the sequence of files
handled, pointing to a pickled ngamsFileInfo object.
Each of the File Info Objects indicates if the file was registered or not.
This is done by setting the tag of the File Info Object to one of the
following values:
REGISTERED: The file was successfully registered in the NGAS DB
FAILED: The file was selected for registration but could not
be properly registered because of inconsistencies.
The status will be of the format: 'FAILED[: <reason>]'.
REJECTED: A file found under the specified path directory was
not accepted for cloning, usually because the mime-type
was not correct. The status will be of the format:
'REJECTED[: <reason>]'.
Note, that registration is comparable to archiving of files. For that
reason a DAPI must be provided for each type of file that should be
registered. If this is not fullfilled, the file registration will
fail.
Only files stored on one of the NGAS disks configured in the
configuration file, are considered. Files stored in other locations
are ignored.
srvObj: Instance of NG/AMS Server object (ngamsServer).
fileListDbmName: Name of a DBM containing the information
about the files to be registered. Each element in the
list is referred to by a key, which is a number.
These points to a pickled list for each file containing
the following information:
[<Filename>, <Disk ID>, <Mime-Type>]
The information for each disk concerned is also contained
in the DB referred to by its Disk ID and by its mount
point. The data is a pickled instance of the ngamsDiskInfo
class (string).
tmpFilePat: Pattern for temporary files used during the registration
process (string).
diskInfoDic: Dictionary with Disk IDs as keys pointing to the info
about the disk (dictionary/ngamsDiskInfo).
reqPropsObj: If an NG/AMS Request Properties Object is given, the
Request Status will be updated as the request is carried
out (ngamsReqProps).
Returns: Void.
"""
T = TRACE()
emailNotif = 0
if (reqPropsObj):
if (reqPropsObj.hasHttpPar("notif_email")):
emailNotif = 1
# Create the temporary BSD DB to contain the information for the
# Email Notification Message.
if (emailNotif):
regDbmName = tmpFilePat + "_NOTIF_EMAIL"
regDbm = ngamsDbm.ngamsDbm(regDbmName, writePerm = 1)
# Open the DBM containing the list of files to (possibly) register.
fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm = 1)
# Want to parse files in alphabetical order.
# TODO: Portatibility issue. Try to avoid UNIX shell commands for sorting.
tmpFileList = tmpFilePat + "_FILE_LIST"
rmFile(tmpFileList)
fo = open(tmpFileList, "w")
fileListDbm.initKeyPtr()
while (1):
dbmKey, fileInfo = fileListDbm.getNext()
if (not dbmKey): break
fo.write(dbmKey + "\n")
fo.close()
sortFileList = tmpFilePat + "_SORT_FILE_LIST"
rmFile(sortFileList)
shellCmd = "sort %s > %s" % (tmpFileList, sortFileList)
stat, out = commands.getstatusoutput(shellCmd)
if (stat != 0):
raise Exception, "Error executing command: %s. Error: %s" %\
(shellCmd, str(out))
rmFile(tmpFileList)
# Go through each file in the list, check if the mime-type is among the
# ones, which apply for registration. If yes try to register the file
# by invoking the corresponding DAPI on the file.
checksumPlugIn = srvObj.getCfg().getChecksumPlugIn().strip()
fileRegCount = 0
fileFailCount = 0
fileRejectCount = 0
regTimeAccu = 0.0
fileCount = 0
fo = open(sortFileList)
run = 1
while (run):
reg_start = time.time()
dbmKey = fo.readline()
if (dbmKey.strip() == ""):
run = 0
continue
fileInfo = fileListDbm.get(dbmKey[0:-1])
filename = fileInfo[0]
diskId = fileInfo[1]
mimeType = fileInfo[2]
# Register the file. Check first, that exactly this file is
# not already registered. In case it is, the file will be rejected.
regPi = srvObj.getCfg().register_plugins[mimeType]
logger.debug("Plugin found for %s: %s", mimeType, regPi)
params = ngamsPlugInApi.parseRawPlugInPars(regPi.pars)
tmpReqPropsObj = ngamsReqProps.ngamsReqProps().\
setMimeType(mimeType).\
setStagingFilename(filename).\
setTargDiskInfo(diskInfoDic[diskId]).\
setHttpMethod(NGAMS_HTTP_GET).\
setCmd(NGAMS_REGISTER_CMD).\
setSize(os.path.getsize(filename)).\
setFileUri(filename).\
setNoReplication(1)
tmpFileObj = ngamsFileInfo.ngamsFileInfo()
try:
# Invoke Registration Plug-In.
piName = regPi.name
plugInMethod = loadPlugInEntryPoint(piName)
piRes = plugInMethod(srvObj, tmpReqPropsObj, params)
del tmpReqPropsObj
# Check if this file is already registered on this disk. In case
# yes, it is not registered again.
files = srvObj.db.getFileSummary1(srvObj.getHostId(), [piRes.getDiskId()],
[piRes.getFileId()])
fileRegistered = 0
for tmpFileInfo in files:
tmpMtPt = tmpFileInfo[ngamsDbCore.SUM1_MT_PT]
tmpFilename = tmpFileInfo[ngamsDbCore.SUM1_FILENAME]
tmpComplFilename = os.path.normpath(tmpMtPt + "/" +\
tmpFilename)
if (tmpComplFilename == filename):
fileRegistered = 1
break
if (fileRegistered):
fileRejectCount += 1
tmpMsgForm = "REJECTED: File with File ID/Version: %s/%d " +\
"and path: %s is already registered on disk " +\
"with Disk ID: %s"
tmpMsg = tmpMsgForm % (piRes.getFileId(),
piRes.getFileVersion(), filename,
piRes.getDiskId())
logger.warning(tmpMsg + ". File is not registered again.")
if (emailNotif):
tmpFileObj.\
setDiskId(diskId).setFilename(filename).\
setTag(tmpMsg)
regDbm.addIncKey(tmpFileObj)
if (reqPropsObj):
reqPropsObj.incActualCount(1)
ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj,
regTimeAccu)
regTimeAccu += time.time() - reg_start
fileCount += 1
continue
# Calculate checksum (if plug-in specified).
if (checksumPlugIn != ""):
logger.debug("Invoking Checksum Plug-In: %s to handle file: %s",
checksumPlugIn, filename)
plugInMethod = loadPlugInEntryPoint(checksumPlugIn)
checksum = plugInMethod(srvObj, filename, 0)
else:
checksum = ""
# Move file and update information about file in the NGAS DB.
mvFile(filename, piRes.getCompleteFilename())
ngamsArchiveUtils.updateFileInfoDb(srvObj, piRes, checksum,
checksumPlugIn)
ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), piRes)
ngamsLib.makeFileReadOnly(piRes.getCompleteFilename())
if (emailNotif):
uncomprSize = piRes.getUncomprSize()
ingestDate = time.time()
creDateSecs = getFileCreationTime(filename)
tmpFileObj.\
setDiskId(diskId).\
setFilename(filename).\
setFileId(piRes.getFileId()).\
setFileVersion(piRes.getFileVersion()).\
setFormat(piRes.getFormat()).\
setFileSize(piRes.getFileSize()).\
setUncompressedFileSize(uncomprSize).\
setCompression(piRes.getCompression()).\
setIngestionDate(ingestDate).\
setIgnore(0).\
setChecksum(checksum).\
setChecksumPlugIn(checksumPlugIn).\
setFileStatus(NGAMS_FILE_STATUS_OK).\
setCreationDate(creDateSecs).\
setTag("REGISTERED")
fileRegCount += 1
# If running as a cache archive, update the Cache New Files DBM
# with the information about the new file.
if (srvObj.getCachingActive()):
fileVer = fio.getFileVersion()
ngamsCacheControlThread.addEntryNewFilesDbm(srvObj, diskId,
piRes.getFileId(),
fileVer, filename)
# Generate a confirmation log entry.
msg = genLog("NGAMS_INFO_FILE_REGISTERED",
[filename, piRes.getFileId(), piRes.getFileVersion(),
piRes.getFormat()])
time.sleep(0.005)
regTime = time.time() - reg_start
msg = msg + ". Time: %.3fs." % (regTime)
logger.info(msg, extra={'to_syslog': 1})
except Exception, e:
errMsg = genLog("NGAMS_ER_FILE_REG_FAILED", [filename, str(e)])
logger.error(errMsg)
if (emailNotif):
tmpFileObj.\
setDiskId(diskId).setFilename(filename).\
setTag(errMsg)
fileFailCount += 1
regTime = time.time() - reg_start
# TODO (rtobar, 2016-01): Why don't we raise an exception here?
# Otherwise the command appears as successful on the
# client-side
# Add the file information in the registration report.
if (emailNotif): regDbm.addIncKey(tmpFileObj)
# Update request status time information.
regTimeAccu += regTime
if (reqPropsObj):
reqPropsObj.incActualCount(1)
ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj,
regTimeAccu)
fileCount += 1
def _cloneExec(srvObj,
cloneListDbmName,
tmpFilePat,
targetDiskId,
reqPropsObj):
"""
See documentation of ngamsCloneCmd._cloneThread(). This function is
merely implemented in order to encapsulate the whole process to be able
to clean up properly when the processing is terminated.
"""
cloneStatusDbm = None
emailNotif = 0
checkChecksum = 1
if (reqPropsObj):
if (reqPropsObj.hasHttpPar("notif_email")): emailNotif = 1
if (reqPropsObj.hasHttpPar("check")):
checkChecksum = int(reqPropsObj.getHttpPar("check"))
# Open clone list DB.
cloneListDbm = ngamsDbm.ngamsDbm(cloneListDbmName)
# We have to get the port numbers of the hosts where the files to be
# cloned are stored.
hostInfoDic = {}
cloneListDbm.initKeyPtr()
while (1):
key, fileInfo = cloneListDbm.getNext()
if (not key): break
hostInfoDic[fileInfo[1]] = -1
hostInfoDic = ngamsHighLevelLib.resolveHostAddress(srvObj.getHostId(),
srvObj.getDb(),
srvObj.getCfg(),
hostInfoDic.keys())
# The cloning loop. Loop over the list of files to clone and generate
# a report with the result.
if (emailNotif):
cloneStatusDbmName = tmpFilePat + "_CLONE_STATUS_DB"
cloneStatusDbm = ngamsDbm.ngamsDbm(cloneStatusDbmName,
cleanUpOnDestr = 0, writePerm = 1)
successCloneCount = 0
failedCloneCount = 0
abortCloneLoop = 0
timeAccu = 0.0
key = 0
while (1):
clone_start = time.time()
if (not cloneListDbm.hasKey(str(key))): break
fileInfo = cloneListDbm.get(str(key))
key += 1
# Check if we have permission to run. Otherwise, stop.
if (not srvObj.run_async_commands): break
fio = fileInfo[0]
mtPt = fileInfo[2]
if (emailNotif):
tmpFileList = ngamsFileList.\
ngamsFileList("FILE_CLONE_STATUS",
"File: " + fio.getFileId() + "/" +\
fio.getDiskId() + "/" +\
str(fio.getFileVersion()))
hostId = fileInfo[1]
text = "Cloning file - File ID: %s/%d, on disk " +\
"with ID: %s on host: %s"
logger.debug(text, fio.getFileId(), fio.getFileVersion(),
fio.getDiskId(), hostId)
# We generate a local Staging File and archive this.
stagingFilename = ""
try:
# Check if file is marked as bad.
if (fio.getFileStatus()[0] == "1"):
errMsg = "File marked as bad - skipping!"
raise Exception(errMsg)
if (targetDiskId == ""):
# Try to find a disk not hosting already a file with that
# ID + version.
diskExemptList = [fio.getDiskId()]
while (1):
trgDiskInfo = ngamsDiskUtils.\
findTargetDisk(srvObj.getHostId(),
srvObj.getDb(),
srvObj.getCfg(),
fio.getFormat(),
1, diskExemptList)
# Check if a file with that ID + version is already
# stored on the selected Target Disk.
if (srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(),
fio.getFileId(),
fio.getFileVersion())):
# This file is already stored on the given disk.
# Add to the exempt list.
diskExemptList.append(trgDiskInfo.getDiskId())
else:
# OK, this disk should be OK, stop looking for a
# suitable Target Disk.
break
else:
try:
trgDiskInfo = ngamsDiskInfo.ngamsDiskInfo().\
read(srvObj.getDb(), targetDiskId)
slotId = trgDiskInfo.getSlotId()
storageSetId = srvObj.getCfg().\
getStorageSetFromSlotId(slotId).\
getStorageSetId()
trgDiskInfo.setStorageSetId(storageSetId)
except Exception:
abortCloneLoop = 1
raise
# We don't accept to clone onto the same disk (this would mean
# overwriting).
if (trgDiskInfo.getDiskId() == fio.getDiskId()):
err = "Source and target files are identical"
msg = "Failed in cloning file with ID: " + fio.getFileId() +\
"/Version: " + str(fio.getFileVersion()) +\
" on disk with ID: " + fio.getDiskId() +\
" on host: " + hostId + ". Reason: " + err
logger.warning(msg)
if (emailNotif):
tmpFileList.setStatus(NGAMS_FAILURE + ": " + err)
tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE"))
cloneStatusDbm.addIncKey(tmpFileList)
failedCloneCount += 1
continue
tmpReqPropsObj = ngamsReqProps.ngamsReqProps()
tmpReqPropsObj.setMimeType(fio.getFormat())
stagingFilename = ngamsHighLevelLib.\
genStagingFilename(srvObj.getCfg(),
tmpReqPropsObj,
trgDiskInfo, fio.getFileId())
# Receive the data into the Staging File using the urllib.
if (srvObj.getHostId() != hostId):
# Example: http://host:7777/RETRIEVE?file_id=id&file_version=1
ipAddress = hostInfoDic[hostId].getIpAddress()
portNo = hostInfoDic[hostId].getSrvPort()
fileUrl = "http://" + ipAddress + ":" + str(portNo) +\
"/RETRIEVE?" + "file_id=" + fio.getFileId() +\
"&file_version=" + str(fio.getFileVersion())
# If a specific Disk ID for the source file is given, append
# this.
if (fio.getDiskId()):
fileUrl += "&disk_id=%s" % fio.getDiskId()
# Check if host is suspended, if yes, wake it up.
if (srvObj.getDb().getSrvSuspended(hostId)):
logger.debug("Clone Request - Waking up suspended " +\
"NGAS Host: %s", hostId)
ngamsSrvUtils.wakeUpHost(srvObj, hostId)
else:
fileUrl = "file:" + mtPt + "/" + fio.getFilename()
logger.debug("Receiving file via URI: %s into staging filename: %s",
fileUrl, stagingFilename)
# We try up to 5 times to retrieve the file in case a problem is
# encountered during cloning.
for attempt in range(5):
try:
filename, headers = urlrequest.urlretrieve(fileUrl, stagingFilename)
_checkFile(srvObj, fio, stagingFilename, headers,
checkChecksum)
# If we get to this point the transfer was (probably) OK.
break
except Exception as e:
rmFile(stagingFilename)
errMsg = "Problem occurred while cloning file "+\
"via URL: " + fileUrl + " - Error: " + str(e)
if (attempt < 4):
errMsg += " - Retrying in 5s ..."
logger.error(errMsg)
time.sleep(5)
else:
raise Exception(errMsg)
# We simply copy the file into the same destination as the
# source file (but on another disk).
targPathName = os.path.dirname(fio.getFilename())
targFilename = os.path.basename(fio.getFilename())
complTargPath = os.path.normpath(trgDiskInfo.getMountPoint() +\
"/" + targPathName)
checkCreatePath(complTargPath)
complFilename = os.path.normpath(complTargPath + "/"+targFilename)
mvTime = mvFile(stagingFilename, complFilename)
ngamsLib.makeFileReadOnly(complFilename)
# Update status for new file in the DB.
newFileInfo = fio.clone().setDiskId(trgDiskInfo.getDiskId()).\
setCreationDate(getFileCreationTime(complFilename))
fileExists = srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(),
fio.getFileId(),
fio.getFileVersion())
newFileInfo.write(srvObj.getHostId(), srvObj.getDb())
# Update status for the Target Disk in DB + check if the disk is
# completed.
if (fileExists): mvTime = 0
dummyDapiStatObj = ngamsDapiStatus.ngamsDapiStatus().\
setDiskId(trgDiskInfo.getDiskId()).\
setFileExists(fileExists).\
setFileSize(fio.getFileSize()).setIoTime(mvTime)
ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), dummyDapiStatObj)
ngamsArchiveUtils.checkDiskSpace(srvObj, trgDiskInfo.getDiskId())
# Update the clone file status list.
if (emailNotif):
tmpFileList.setStatus(NGAMS_SUCCESS)
tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE"))
tmpFileList.addFileInfoObj(newFileInfo.setTag("TARGET_FILE"))
cloneStatusDbm.addIncKey(tmpFileList)
successCloneCount += 1
# If running as a cache archive, update the Cache New Files DBM
# with the information about the new file.
if (srvObj.getCachingActive()):
diskId = trgDiskInfo.getDiskId()
fileId = fio.getFileId()
fileVer = fio.getFileVersion()
filename = fio.getFilename()
ngamsCacheControlThread.addEntryNewFilesDbm(srvObj, diskId,
fileId, fileVer,
filename)
# Generate a confirmation log entry.
cloneTime = time.time() - clone_start
timeAccu += cloneTime
msg = genLog("NGAMS_INFO_FILE_CLONED",
[fio.getFileId(), fio.getFileVersion(),
fio.getDiskId(), hostId])
msg = msg + ". Time: %.3fs. Total time: %.3fs." %\
(cloneTime, timeAccu)
logger.info(msg, extra={'to_syslog': True})
except Exception as e:
cloneTime = time.time() - clone_start
timeAccu += cloneTime
errMsg = genLog("NGAMS_ER_FILE_CLONE_FAILED",
[fio.getFileId(), fio.getFileVersion(),
fio.getDiskId(), hostId, str(e)])
if (abortCloneLoop):
logger.error(errMsg, extra={'to_syslog': True})
return
else:
logger.warning(errMsg)
if (emailNotif):
tmpFileList.setStatus(NGAMS_FAILURE + ": Error: " + errMsg)
tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE"))
cloneStatusDbm.addIncKey(tmpFileList)
failedCloneCount += 1
# Delete Staging File if already created.
if ((stagingFilename != "") and (os.path.exists(stagingFilename))):
rmFile(stagingFilename)
# Calculate time statistics.
if (reqPropsObj):
ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj.\
incActualCount(1), timeAccu)
# Final update of the Request Status.
if (reqPropsObj):
complPercent = (100.0 * (float(reqPropsObj.getActualCount()) /
float(reqPropsObj.getExpectedCount())))
reqPropsObj.setCompletionPercent(complPercent, 1)
reqPropsObj.setCompletionTime(1)
srvObj.updateRequestDb(reqPropsObj)
# Send Clone Report with list of files cloned to a possible
# requestor(select) of this.
totFiles = (successCloneCount + failedCloneCount)
if (emailNotif):
xmlStat = 0
# TODO: Generation of XML status report is disabled since we cannot
# handle for the moment XML documents with 1000s of elements.
if (xmlStat):
cloneStatusFileList = ngamsFileList.\
ngamsFileList("FILE_CLONING_STATUS_REPORT",
"File Cloning Status Report")
fileCount = 0
while (fileCount < cloneStatusDbm.getCount()):
tmpFileList = cloneStatusDbm.get(str(fileCount))
cloneStatusFileList.addFileListObj(tmpFileList)
# Make overall status.
cloneStatusFileList.setStatus("SUCCESS: " +\
str(successCloneCount) +\
", FAILURE: " +\
str(failedCloneCount) +\
", NOT DONE: " +\
str(len(cloneStatusFileList) -\
successCloneCount -\
failedCloneCount))
status = srvObj.genStatus(NGAMS_SUCCESS,
"CLONE command status report").\
addFileList(cloneStatusFileList)
statRep = status.genXmlDoc(0, 0, 0, 1, 0)
statRep = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, statRep)
mimeType = NGAMS_XML_MT
else:
# Generate a 'simple' ASCII report.
statRep = tmpFilePat + "_NOTIF_EMAIL.txt"
fo = open(statRep, "w")
if (reqPropsObj.hasHttpPar("disk_id")):
diskId = reqPropsObj.getHttpPar("disk_id")
else:
diskId = "-----"
if (reqPropsObj.hasHttpPar("file_id")):
fileId = reqPropsObj.getHttpPar("file_id")
else:
fileId = "-----"
if (reqPropsObj.hasHttpPar("file_version")):
fileVersion = reqPropsObj.getHttpPar("file_version")
else:
fileVersion = "-----"
tmpFormat = "CLONE STATUS REPORT:\n\n" +\
"==Summary:\n\n" +\
"Date: %s\n" +\
"NGAS Host: %s\n" +\
"Disk ID: %s\n" +\
"File ID: %s\n" +\
"File Version: %s\n" +\
"Total Number of Files: %d\n" +\
"Number of Cloned Files: %d\n" +\
"Number of Failed Files: %d\n" +\
"Total processing time (s): %.3f\n" +\
"Handling time per file (s): %.3f\n\n" +\
"==File List:\n\n"
fo.write(tmpFormat % (toiso8601(), srvObj.getHostId(), diskId, fileId,
str(fileVersion), totFiles,
successCloneCount, failedCloneCount,
timeAccu, (timeAccu / totFiles)))
tmpFormat = "%-70s %-70s %-7s\n"
fo.write(tmpFormat % ("Source File", "Target File", "Status"))
fo.write(tmpFormat % (70 * "-", 70 * "-", 7 * "-"))
key = 1
while (1):
if (not cloneStatusDbm.hasKey(str(key))): break
tmpFileList = cloneStatusDbm.get(str(key))
key += 1
srcFileObj = tmpFileList.getFileInfoObjList()[0]
srcFile = "%s/%s/%d" % (srcFileObj.getDiskId(),
srcFileObj.getFileId(),
srcFileObj.getFileVersion())
if (tmpFileList.getStatus() == NGAMS_SUCCESS):
trgFileObj = tmpFileList.getFileInfoObjList()[1]
trgFile = "%s/%s/%d" % (trgFileObj.getDiskId(),
trgFileObj.getFileId(),
trgFileObj.getFileVersion())
else:
trgFile = "-----"
fo.write(tmpFormat % (srcFile,trgFile,tmpFileList.getStatus()))
fo.write(149 * "-")
fo.write("\n\n==END\n")
fo.close()
mimeType = NGAMS_TEXT_MT
# Send out the status report.
emailAdrList = reqPropsObj.getHttpPar("notif_email").split(",")
attachmentName = "CloneStatusReport"
if (reqPropsObj.hasHttpPar("disk_id")):
attachmentName += "-" + reqPropsObj.getHttpPar("disk_id")
if (reqPropsObj.hasHttpPar("file_id")):
attachmentName += "-" + reqPropsObj.getHttpPar("file_id")
if (reqPropsObj.hasHttpPar("file_version")):
attachmentName += "-" + reqPropsObj.getHttpPar("file_version")
ngamsNotification.notify(srvObj.host_id, srvObj.cfg, NGAMS_NOTIF_INFO,
"CLONE STATUS REPORT", statRep, recList=emailAdrList, force=1,
contentType=mimeType, attachmentName=attachmentName)
del cloneStatusDbm
rmFile(cloneStatusDbmName + "*")
rmFile(statRep)
if (cloneListDbm): del cloneListDbm
rmFile(cloneListDbmName + "*")
logger.info("_cloneExec(). Total time: %.3fs. Average time per file: %.3fs.",
timeAccu, (timeAccu / totFiles))
def _handleFileListReply(srvObj,
reqPropsObj,
httpRef,
fileListId,
maxElements = None):
"""
Extracts file information from a previously dumped file information
in connection with a STATUS?file_list request.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler object
(ngamsHttpRequestHandler).
fileListId: File List ID allocated to this request (string).
maxElements: Maximum number of elements to extract and return to the
requestor (integer).
Returns: Void.
"""
# Get the name of the DBM in which the information is stored.
dbmBaseName = STATUS_FILE_LIST_DBM_TAG % fileListId
dbmPat = os.path.normpath("%s/*%s*" %\
(ngamsHighLevelLib.\
getNgasTmpDir(srvObj.getCfg()),
dbmBaseName))
dbmMatches = glob.glob(dbmPat)
if (len(dbmMatches) < 1):
msg = "Referenced File List ID: %s in connection with " +\
"STATUS/file_list request, is not (or no longer) known"
raise Exception(msg % fileListId)
elif (len(dbmMatches) > 1):
msg = "Inconsistencies encountered in locating result set for " +\
"STATUS/file_list for referenced File List ID: %s"
raise Exception(msg % fileListId)
fileInfoDbmName = dbmMatches[0]
# Generate the NG/AMS Status Document, with a File List in it.
# Compress it on the fly.
fileListXmlDoc = ngamsHighLevelLib.\
genTmpFilename(srvObj.getCfg(),
"STATUS_FILE_LIST_XML.xml")
try:
tmpFileListDbm = ngamsDbm.ngamsDbm(fileInfoDbmName,
cleanUpOnDestr = 0,
writePerm = 1)
rmFile(fileListXmlDoc)
fo = open(fileListXmlDoc, "w")
if (not maxElements):
remainingObjects = 0
else:
remainingObjects = (tmpFileListDbm.getCount() - maxElements)
if (remainingObjects < 0):
remainingObjects = 0
fo.write(_fileListXmlHdr % (fileListId, str(remainingObjects)))
# Loop over the file info objects and write them into the file.
# take max the number of elements specified.
tmpFileListDbm.initKeyPtr()
elCount = 0
keyRefList = []
while (True):
# Have the requested number of elements been extracted?
if (maxElements):
if (elCount >= maxElements):
break
# Get the next key (if there are more elements).
key, fileInfo = tmpFileListDbm.getNext()
if (not key):
break
try:
# Write the File Status XML Element in the file.
tmpFileInfoObj = ngamsFileInfo.ngamsFileInfo().\
unpackSqlResult(fileInfo)
fileInfoXml = tmpFileInfoObj.genXml(storeDiskId = 1).\
toprettyxml(" ", "\n")[:-1]
fo.write("\n" + fileInfoXml)
except Exception as e:
msg = "Error creating STATUS/File List XML Document. " +\
"Error: %s" % str(e)
logger.error(msg)
raise
keyRefList.append(key)
elCount += 1
# Finish up the XML document, close the file.
fo.write(_fileListXmlFooter)
fo.close()
# Assume this type of file can always be compressed.
fileListXmlDoc = compressFile(fileListXmlDoc)
except:
rmFile("%s*" % fileInfoDbmName)
rmFile("%s*" % fileListXmlDoc)
raise
# Send the XML document back to the requestor.
try:
httpRef.send_file(fileListXmlDoc, NGAMS_GZIP_XML_MT)
# Remove the reported entries.
for key in keyRefList:
tmpFileListDbm.rem(key)
tmpFileListDbm.sync()
del keyRefList
keyRefList = []
# If there are no more entries, delete the DBM.
dbmCount = tmpFileListDbm.getCount()
del tmpFileListDbm
if (dbmCount == 0):
rmFile("%s*" % fileInfoDbmName)
except Exception as e:
msg = "Error returning response to STATUS?file_list request. Error: %s"
msg = msg % str(e)
raise Exception(msg)
finally:
rmFile(fileListXmlDoc)
def _dumpFileInfo(srvObj, disks_to_check, tmpFilePat, stopEvt):
"""
Function that dumps the information about the files. One DBM is created
per disk. This is named:
<Mount Root Point>/cache/DATA-CHECK-THREAD_QUEUE_<Disk ID>.bsddb
If problems are found for a file, these are stored in DBM files named:
<Mount Root Point>/cache/DATA-CHECK-THREAD_ERRORS_<Disk ID>.bsddb
The function handles the DBM files in the following way:
1. Check for each DBM file found, if this disk is still in the system.
If not, the Queue and Error DBM files are removed.
2. Go through the list of disks in the system. If they don't have
the two DBM files listed above, these are initialized. The file
information for all the files stored on the disk is dumped into
the Queue DBM file. Only files marked to be ignored are not dumped.
3. Finally, build up a DBM with references to all files found
on this system
srvObj: Reference to server object (ngamsServer).
tmpFilePat: Pattern for temporary files (string).
Returns: Void.
"""
T = TRACE()
cacheDir = os.path.join(srvObj.getCfg().getRootDirectory(),
NGAMS_CACHE_DIR)
checkCreatePath(os.path.normpath(cacheDir))
###########################################################################
# Loop over the Queue/Error DBM files found, check if the disk is
# still in the system/scheduled for checking.
###########################################################################
logger.debug("Loop over/check existing Queue/Error DBM Files ...")
dbmFileList = glob.glob(cacheDir + "/" + NGAMS_DATA_CHECK_THR +\
"_QUEUE_*.bsddb")
dbmObjDic = {}
for dbmFile in dbmFileList:
_stopDataCheckThr(stopEvt)
diskId = dbmFile.split("_")[-1].split(".")[0]
if diskId not in disks_to_check:
filePat = "%s/%s*%s.bsddb" % (cacheDir, NGAMS_DATA_CHECK_THR,
diskId)
rmFile(filePat)
else:
# Add references to Queue/Error DBM.
queueDbmFile = "%s/%s_QUEUE_%s.bsddb" %\
(cacheDir, NGAMS_DATA_CHECK_THR, diskId)
queueDbm = ngamsDbm.ngamsDbm(queueDbmFile, 0, 1)
errorDbmFile = "%s/%s_ERRORS_%s.bsddb" %\
(cacheDir, NGAMS_DATA_CHECK_THR, diskId)
errorDbm = ngamsDbm.ngamsDbm(errorDbmFile, 0, 1)
dbmObjDic[diskId] = (queueDbm, errorDbm)
logger.debug("Looped over/checked existing Queue/Error DBM Files")
###########################################################################
###########################################################################
# Loop over the disks mounted in this system and check if they have a
# Queue/Error DBM file. In case the DBM files are not available, create
# these.
###########################################################################
logger.debug("Create DBM files for disks to be checked ...")
startDbFileRd = time.time()
for diskId in disks_to_check.keys():
_stopDataCheckThr(stopEvt)
if diskId in dbmObjDic:
continue
# The disk is ripe for checking but still has no Queue/Error DBM
# DBs allocated.
queueDbmFile = "%s/%s_QUEUE_%s.bsddb" %\
(cacheDir, NGAMS_DATA_CHECK_THR, diskId)
tmpQueueDbmFile = tmpFilePat + "_" + os.path.basename(queueDbmFile)
queueDbm = ngamsDbm.ngamsDbm(tmpQueueDbmFile, 0, 1)
# Now, retrieve the files on the given disk, and store the info
# in the Queue DBM file.
files = srvObj.getDb().getFileSummary1(diskIds=[diskId],
ignore=0,
fileStatus=[],
lowLimIngestDate=None,
order=0)
for fileInfo in files:
fileId = fileInfo[ngamsDbCore.SUM1_FILE_ID]
fileVer = fileInfo[ngamsDbCore.SUM1_VERSION]
fileKey = ngamsLib.genFileKey(None, fileId, fileVer)
queueDbm.add(fileKey, fileInfo)
queueDbm.sync()
# Rename DCC Queue DBM from the temporary to the final name.
mvFile(tmpQueueDbmFile, queueDbmFile)
queueDbm = ngamsDbm.ngamsDbm(queueDbmFile, 0, 1)
# Create Error DBM + add these in the DBM Dictionary for the disk.
errorDbmFile = "%s/%s_ERRORS_%s.bsddb" %\
(cacheDir, NGAMS_DATA_CHECK_THR, diskId)
errorDbm = ngamsDbm.ngamsDbm(errorDbmFile, 0, 1)
dbmObjDic[diskId] = (queueDbm, errorDbm)
_stopDataCheckThr(stopEvt)
logger.debug("Queried info for files to be checked from DB. Time: %.3fs",
time.time() - startDbFileRd)
logger.debug("Checked that disks scheduled for checking have DBM files")
###########################################################################
# These are all files recursively found on the disks
# Later on we check whether they are registered or not, and check them (or not)
start = time.time()
files_on_disk = collect_files_on_disk(stopEvt, disks_to_check)
end = time.time()
logger.debug("Collected references to %d files on disks in %.3f [s]",
len(files_on_disk), end - start)
# Don't take these into account
logger.debug("Retrieving information about files to be ignored ...")
files = srvObj.getDb().getFileSummarySpuriousFiles1(srvObj.getHostId(),
fetch_size=1000)
for fileInfo in files:
if (fileInfo[ngamsDbCore.SUM1_FILE_IGNORE]):
filename = os.path.\
normpath(fileInfo[ngamsDbCore.SUM1_MT_PT] + "/" +\
fileInfo[ngamsDbCore.SUM1_FILENAME])
if filename in files_on_disk:
del files_on_disk[filename]
logger.debug("Retrieved information about files to be ignored")
###########################################################################
###########################################################################
# Initialize the statistics parameters for the checking.
###########################################################################
logger.debug("Initialize the statistics for the checking cycle ...")
amountMb = 0.0
noOfFiles = 0
for diskId in disks_to_check.keys():
queueDbm = dbmObjDic[diskId][0]
#################################################################################################
#jagonzal: Replace looping aproach to avoid exceptions coming from the next() method underneath
# when iterating at the end of the table that are prone to corrupt the hash table object
#queueDbm.initKeyPtr()
#while (1):
# fileKey, fileInfo = queueDbm.getNext()
# if (not fileKey): break
for fileKey, dbVal in queueDbm.iteritems():
# jagonzal: We need to reformat the values and skip administrative elements #################
if (str(fileKey).find("__") != -1): continue
fileInfo = cPickle.loads(dbVal)
#############################################################################################
noOfFiles += 1
amountMb += float(fileInfo[ngamsDbCore.SUM1_FILE_SIZE]) / 1048576.0
#################################################################################################
stats = _initFileCheckStatus(srvObj, amountMb, noOfFiles)
###########################################################################
return files_on_disk, dbmObjDic, stats
def checkFileCopiesAndReg(srvObj,
minReqCopies,
dbFilePat,
fileListDbmName=None,
diskId=None,
ignoreMounted=0):
"""
The function checks for each file referenced if there are at least
'minReqCopies' copies available somewhere in this NGAS cluster. For the
files where this is not the case, an entry is added in a ngasDiskInfo
object indicating that this file
If an entire disk is analysed (about to be deleted), it is also checked if
each file stored on the disk is registered in the DB. Otherwise an NG/AMS
File List is returned, containing references to the files not registered.
srvObj: Instance of the NG/AMS Server Class (ngamsServer).
minReqCopies: Minimum number of copies required (integer).
dbFilePat: Filename pattern used to build the DBM containing
information about the files (string).
fileListDbmName: Name of DBM DB containing explicit references to files
to be checked if they can be deleted. The information
in this table is pickled lists with the lay-out defined
by ngamsDb._ngasFilesCols (string).
diskId: Used to refer to all files stored on a disk (string|None).
ignoreMounted: Carry out the check also if the disk is not mounted
(integer/0|1).
Returns: Tuple contaning the filenames of three DBM DBs with the
following information:
o Files not having the specified number of copies.
The contents of this DB are keys (Disk ID + File ID +
File Version), pointing to pickled ngamsFileInfo
objects.
o Files found on the disk but not registered.
This contains the complete filenames of files found
on the disk, which are not registered. These filenames
are the keys of this DBM DB.
o Complete list of files referenced in connection
with the query. The contents of this DB are keys,
which are a simple counter pointing to pickled
list containing the information as returned by
ngamsDb.getFileSummary1().
(tuple/string).
"""
T = TRACE()
if ((not fileListDbmName) and (not diskId)):
errMsg = "ngamsSrvUtils.checkFileCopiesAndReg(): Must specify " +\
"either a DBM with files to be checked or a Disk ID"
logger.warning(errMsg)
raise Exception(errMsg)
# Create DBMs:
# - DB containing information about files having less then the
# specified number of copies.
fileMisCopyDbmName = os.path.normpath(dbFilePat + "_MISSING_COPIES")
fileMisCopyDbm = ngamsDbm.ngamsDbm(fileMisCopyDbmName, writePerm=1)
# - DB that contains information about files stored on the DB,
# which are not registered in the NGAS DB. At the end of the function,
# this will contain information about files found on the disk but
# not registered in the NGAS DB.
filesOnDiskDicDbmName = os.path.normpath(dbFilePat + "_FILES_ON_DISK")
filesOnDiskDicDbm = ngamsDbm.ngamsDbm(filesOnDiskDicDbmName, writePerm=1)
# - DB with information about files referenced by the query.
if (not fileListDbmName):
locFileListDbmName = os.path.normpath(dbFilePat + "_FILE_LIST")
fileListDbm = ngamsDbm.ngamsDbm(locFileListDbmName, writePerm=1)
else:
fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm=0)
# - Temporary DBM containing information about all File IDs defined
# by the query.
fileIdDbmName = os.path.normpath(dbFilePat + "_FILE_IDS")
fileIdDbm = ngamsDbm.ngamsDbm(fileIdDbmName, writePerm=1)
# - Temporary DBM containing information about all files available in
# the system with the File ID/File Version defined by the query.
complFileListDbmName = os.path.normpath(dbFilePat + "_COMPL_FILE_LIST")
complFileListDbm = ngamsDbm.ngamsDbm(complFileListDbmName, writePerm=1)
# - Temporary DBM that is used to figure out the number of independent
# copies of each file concerned by the query.
checkDicDbmName = os.path.normpath(dbFilePat + "_CHECK_DIC")
checkDicDbm = ngamsDbm.ngamsDbm(checkDicDbmName, writePerm=1)
# A Disk ID but no file references are given. Retrieve information
# about files concerned from the DB.
if (diskId):
logger.debug("Retrieving information about files on disk with ID: %s",
diskId)
for f in srvObj.db.getFileSummary1(None, [diskId], [],
ignore=0,
fileStatus=[]):
fileListDbm.addIncKey(f)
fileId = f[ngamsDbCore.SUM1_FILE_ID]
fileIdDbm.add(str(fileId), "")
fileVersion = f[ngamsDbCore.SUM1_VERSION]
fileKey = ngamsLib.genFileKey(None, fileId, fileVersion)
checkDicDbm.add(fileKey, {})
fileListDbm.sync()
fileIdDbm.sync()
checkDicDbm.sync()
# Get the list of files located on the disk. Later on, remove entries
# from this dictionary as the files are parsed, based on their DB info,
# further down in this method.
#
# Key in this dictionary is the complete filename of the file.
logger.debug("Get list of files stored on disk ...")
tmpDiskInfo = srvObj.getDb().getDiskInfoFromDiskId(diskId)
diskInfoObj = ngamsDiskInfo.ngamsDiskInfo().\
unpackSqlResult(tmpDiskInfo)
if ((not ignoreMounted) and (not diskInfoObj.getMounted())):
errMsg = "Rejecting request for removing disk with ID: " +\
diskId + " - disk not mounted!"
raise Exception(errMsg)
if (not ignoreMounted):
basePath = os.path.normpath(diskInfoObj.getMountPoint())
pattern = "/*"
logger.debug(
"Generating list with files on disk with base path: %s",
basePath)
while (1):
tmpFileList = glob.glob(basePath + pattern)
if (len(tmpFileList) == 0):
break
else:
for filename in tmpFileList:
if (os.path.isfile(filename) and
(os.path.basename(filename) != NGAMS_DISK_INFO)
and (os.path.basename(filename) !=
NGAMS_VOLUME_ID_FILE)
and (os.path.basename(filename) !=
NGAMS_VOLUME_INFO_FILE)):
filesOnDiskDicDbm.add(str(filename), "")
pattern += "/*"
# Generate File ID DBM in case a file list DBM is given.
if (fileListDbmName):
logger.debug("Handling file list DBM given in the function call ...")
fileListDbm.initKeyPtr()
while (1):
key, tmpFileInfo = fileListDbm.getNext()
if (not key): break
# Update the File ID DBM.
fileId = tmpFileInfo[ngamsDbCore.SUM1_FILE_ID]
fileIdDbm.add(str(fileId), "")
# Update the DBM with references to File ID/Version sets.
fileVersion = tmpFileInfo[ngamsDbCore.SUM1_VERSION]
fileKey = ngamsLib.genFileKey(None, fileId, fileVersion)
checkDicDbm.add(fileKey, {})
fileIdDbm.sync()
checkDicDbm.sync()
# We need to generate a list with all files available in the system
# with the given File ID/File Version.
logger.debug("Retrieving information about all files available with the " +\
"File ID/File Version as defined by the query")
# Due to the limitation of the size of SQL queries, we have to split up
# the SQL query in several sub-queries. The max. length of an SQL query
# is defined by NGAMS_MAX_SQL_QUERY_SZ, we subtract 512 from this for
# the general part of the query, and for each filename we calculate a
# length of len(File ID) + 4 as contribution to the SQL query.
maxQuerySize = (NGAMS_MAX_SQL_QUERY_SZ - 512)
queryIds = []
querySize = 0
noOfFileIds = fileIdDbm.getCount()
fileIdCount = 0
fileIdDbm.initKeyPtr()
fileId = "INIT"
while (fileId):
fileId, dummy = fileIdDbm.getNext()
if (fileId):
queryIds.append(utils.b2s(fileId))
fileIdCount += 1
querySize += (len(fileId) + 4)
if queryIds and (querySize >= maxQuerySize
or fileIdCount == noOfFileIds):
for f in srvObj.db.getFileSummary1(fileIds=queryIds,
fileStatus=[]):
# Take only a sub-result if that File ID + Version
# is concerned by the query.
tmpFileId = f[ngamsDbCore.SUM1_FILE_ID]
tmpFileVersion = f[ngamsDbCore.SUM1_VERSION]
tmpFileKey = ngamsLib.genFileKey(None, tmpFileId,
tmpFileVersion)
if (checkDicDbm.hasKey(tmpFileKey)):
complFileListDbm.addIncKey(f)
complFileListDbm.sync()
queryIds = []
querySize = 0
# Now, go through the files found and order these such that we end up with
# a Dictionary with "<File ID>_<File Version>" as keys referring
# to a dictionary with the Disk IDs of the disks hosting the files as
# keys, and the information for each file on that disk as a tupple.
#
# It is the intention to figure out how many copies we have of each file
# identified by File ID + File Version stored ON DIFFERENT STORAGE MEDIAS
# + on different hosts.
logger.debug("Generate DBM DB with info about independent file copies ...")
complFileListDbm.initKeyPtr()
while (1):
fileKey, fileInfo = complFileListDbm.getNext()
if (not fileKey): break
checkDicKey = ngamsLib.genFileKey(None,
fileInfo[ngamsDbCore.SUM1_FILE_ID],
fileInfo[ngamsDbCore.SUM1_VERSION])
tmpDic = checkDicDbm.get(checkDicKey)
tmpDic[fileInfo[ngamsDbCore.SUM1_DISK_ID]] = fileInfo
checkDicDbm.add(checkDicKey, tmpDic)
# Check if there are at least minReqCopies occurrences of the files +
# check that all files are registered (if a Disk ID is specified).
logger.debug("Check for files with less copies than: %s",
str(minReqCopies))
checkDicDbm.initKeyPtr()
while (1):
checkDicKey, tmpDic = checkDicDbm.getNext()
if (not checkDicKey): break
tmpDicKeys = list(tmpDic)
noOfCopies = len(tmpDicKeys)
if (noOfCopies < minReqCopies):
tmpFileInfo = tmpDic[tmpDicKeys[0]]
fileId = tmpFileInfo[ngamsDbCore.SUM1_FILE_ID]
fileVersion = tmpFileInfo[ngamsDbCore.SUM1_VERSION]
tmpFileObj = ngamsFileInfo.ngamsFileInfo().\
setFileId(fileId).\
setFileVersion(fileVersion).\
setTag("Independent copies: " + str(noOfCopies))
fileKey = ngamsLib.genFileKey(None, fileId, fileVersion)
fileMisCopyDbm.add(str(fileKey), tmpFileObj)
# Remove this file from the Files On Disk DBM - do this only
# if a Disk ID is specified.
if (diskId):
if (diskId in tmpDic):
fileInfo = tmpDic[diskId]
filename = os.path.\
normpath(fileInfo[ngamsDbCore.SUM1_MT_PT] +\
"/" + fileInfo[ngamsDbCore.SUM1_FILENAME])
filename = str(filename)
if (filesOnDiskDicDbm.hasKey(filename)):
filesOnDiskDicDbm.rem(filename)
# Close all DBM objects.
del fileMisCopyDbm
del filesOnDiskDicDbm
del fileListDbm
del fileIdDbm
del complFileListDbm
del checkDicDbm
# The DBM filesOnDiskDicDbmName now contains references to files,
# which are found on the disk but not registered in the DB.
return (fileMisCopyDbmName, filesOnDiskDicDbmName, complFileListDbmName)
def checkUpdateDbSnapShots(srvObj, stopEvt):
"""
Check if a DB Snapshot exists for the DB connected. If not, this is
created according to the contents of the NGAS DB (if possible). During
this creation it is checked if the file are physically stored on the
disk.
srvObj: Reference to NG/AMS server class object (ngamsServer).
Returns: Void.
"""
snapshotDbm = None
tmpSnapshotDbm = None
if (not srvObj.getCfg().getDbSnapshot()):
logger.debug("NOTE: DB Snapshot Feature is switched off")
return
logger.debug("Generate list of disks to check ...")
tmpDiskIdMtPtList = srvObj.getDb().getDiskIdsMtPtsMountedDisks(
srvObj.getHostId())
diskIdMtPtList = []
for diskId, mtPt in tmpDiskIdMtPtList:
diskIdMtPtList.append([mtPt, diskId])
diskIdMtPtList.sort()
logger.debug("Generated list of disks to check: %s", str(diskIdMtPtList))
# Generate temporary snapshot filename.
ngasId = srvObj.getHostId()
tmpDir = ngamsHighLevelLib.getTmpDir(srvObj.getCfg())
# Temporary DBM with file info from the DB.
tmpSnapshotDbmName = os.path.normpath(tmpDir + "/" + ngasId + "_" +\
NGAMS_DB_NGAS_FILES)
# Temporary DBM to contain information about 'lost files', i.e. files,
# which are registered in the DB and found in the DB Snapshot, but
# which are not found on the disk.
logger.debug("Create DBM to hold information about lost files ...")
lostFileRefsDbmName = os.path.normpath(tmpDir + "/" + ngasId +\
"_LOST_FILES")
rmFile(lostFileRefsDbmName + "*")
lostFileRefsDbm = ngamsDbm.ngamsDbm(lostFileRefsDbmName, writePerm=1)
# Carry out the check.
for mtPt, diskId in diskIdMtPtList:
checkStopJanitorThread(stopEvt)
logger.debug("Check/create/update DB Snapshot for disk with " +\
"mount point: %s", mtPt)
try:
snapshotDbm = _openDbSnapshot(srvObj.getCfg(), mtPt)
if (snapshotDbm == None):
continue
# The scheme for synchronizing the Snapshot and the DB is:
#
# - Loop over file entries in the Snapshot:
# - If in DB:
# - If file on disk -> OK, do nothing.
# - If file not on disk -> Accumulate + issue collective warning.
#
# - If entry not in DB:
# - If file on disk -> Add entry in DB.
# - If file not on disk -> Remove entry from Snapshot.
#
# - Loop over entries for that disk in the DB:
# - If entry in Snapshot -> OK, do nothing.
# - If entry not in Snapshot:
# - If file on disk -> Add entry in Snapshot.
# - If file not on disk -> Remove entry from DB.
# Create a temporary DB Snapshot with the files from the DB.
try:
rmFile(tmpSnapshotDbmName + "*")
tmpSnapshotDbm = bsddb.hashopen(tmpSnapshotDbmName, "c")
for fileInfo in srvObj.db.getFileInfoList(diskId, ignore=None):
fileKey = _genFileKey(fileInfo)
encFileInfoDic = _encFileInfo(srvObj.getDb(),
tmpSnapshotDbm, fileInfo)
_addInDbm(tmpSnapshotDbm, fileKey, encFileInfoDic)
checkStopJanitorThread(stopEvt)
tmpSnapshotDbm.sync()
except:
rmFile(tmpSnapshotDbmName)
raise
#####################################################################
# Loop over the possible entries in the DB Snapshot and compare
# these against the DB.
#####################################################################
logger.debug("Loop over file entries in the DB Snapshot - %s ...",
diskId)
count = 0
try:
key, pickleValue = snapshotDbm.first()
except Exception as e:
msg = "Exception raised accessing DB Snapshot for disk: %s. " +\
"Error: %s"
logger.debug(msg, diskId, str(e))
key = None
snapshotDbm.dbc = None
# Create a DBM which is used to keep the list of files to remove
# from the DB Snapshot.
snapshotDelDbmName = ngamsHighLevelLib.\
genTmpFilename(srvObj.getCfg(),
NGAMS_DB_NGAS_FILES)
snapshotDelDbm = ngamsDbm.ngamsDbm(snapshotDelDbmName,
cleanUpOnDestr=1,
writePerm=1)
#################################################################################################
#jagonzal: Replace looping aproach to avoid exceptions coming from the next() method underneath
# when iterating at the end of the table that are prone to corrupt the hash table object
#while (key):
for key, pickleValue in snapshotDbm.iteritems():
#################################################################################################
value = cPickle.loads(pickleValue)
# Check if an administrative element, if yes add it if necessary.
if b"___" in key:
if (not tmpSnapshotDbm.has_key(key)):
tmpSnapshotDbm[key] = pickleValue
else:
tmpFileObj = _encFileInfo2Obj(srvObj.getDb(), snapshotDbm,
value)
if (tmpFileObj is None):
continue
complFilename = os.path.normpath(mtPt + "/" +\
tmpFileObj.getFilename())
# Is the file in the DB?
if (tmpSnapshotDbm.has_key(key)):
# Is the file on the disk?
if (not os.path.exists(complFilename)):
fileVer = tmpFileObj.getFileVersion()
tmpFileObj.setTag(complFilename)
fileKey = ngamsLib.genFileKey(
tmpFileObj.getDiskId(), tmpFileObj.getFileId(),
fileVer)
lostFileRefsDbm.add(fileKey, tmpFileObj)
lostFileRefsDbm.sync()
elif (not tmpSnapshotDbm.has_key(key)):
tmpFileObj = _encFileInfo2Obj(srvObj.getDb(),
snapshotDbm, value)
if (tmpFileObj is None):
continue
# Is the file on the disk?
if (os.path.exists(complFilename)):
# Add this entry in the NGAS DB.
tmpFileObj.write(srvObj.getHostId(),
srvObj.getDb(), 0, 1)
tmpSnapshotDbm[key] = pickleValue
else:
# Remove this entry from the DB Snapshot.
msg = "Scheduling entry: %s in DB Snapshot " +\
"for disk with ID: %s for removal"
logger.debug(msg, diskId, key)
# Add entry in the DB Snapshot Deletion DBM marking
# the entry for deletion.
if (_updateSnapshot(srvObj.getCfg())):
snapshotDelDbm.add(key, 1)
del tmpFileObj
# Be friendly and sync the DB file every now and then
count += 1
if (count % 100) == 0:
if _updateSnapshot(srvObj.getCfg()):
snapshotDbm.sync()
checkStopJanitorThread(stopEvt)
tmpSnapshotDbm.sync()
#################################################################################################
#jagonzal: Replace looping aproach to avoid exceptions coming from the next() method underneath
# when iterating at the end of the table that are prone to corrupt the hash table object
#try:
# key, pickleValue = snapshotDbm.next()
#except:
# key = None
# snapshotDbm.dbc = None
#################################################################################################
# Now, delete entries in the DB Snapshot if there are any scheduled for
# deletion.
#################################################################################################
#jagonzal: Replace looping aproach to avoid exceptions coming from the next() method underneath
# when iterating at the end of the table that are prone to corrupt the hash table object
#snapshotDelDbm.initKeyPtr()
#while (True):
# key, value = snapshotDelDbm.getNext()
# if (not key): break
for key, value in snapshotDelDbm.iteritems():
# jagonzal: We need to reformat the values and skip administrative elements #################
if b'__' in key:
continue
#############################################################################################
msg = "Removing entry: %s from DB Snapshot for disk with ID: %s"
logger.debug(msg, key, diskId)
del snapshotDbm[key]
#################################################################################################
del snapshotDelDbm
logger.debug("Looped over file entries in the DB Snapshot - %s",
diskId)
# End-Loop: Check DB against DB Snapshot. ###########################
if (_updateSnapshot(srvObj.getCfg())): snapshotDbm.sync()
tmpSnapshotDbm.sync()
logger.info(
"Checked/created/updated DB Snapshot for disk with mount point: %s",
mtPt)
#####################################################################
# Loop over the entries in the DB and compare these against the
# DB Snapshot.
#####################################################################
logger.debug("Loop over the entries in the DB - %s ...", diskId)
count = 0
try:
key, pickleValue = tmpSnapshotDbm.first()
except:
key = None
tmpSnapshotDbm.dbc = None
#################################################################################################
#jagonzal: Replace looping aproach to avoid exceptions coming from the next() method underneath
# when iterating at the end of the table that are prone to corrupt the hash table object
#while (key):
for key, pickleValue in tmpSnapshotDbm.iteritems():
#################################################################################################
value = cPickle.loads(pickleValue)
# Check if it is an administrative element, if yes add it if needed
if b"___" in key:
if (not snapshotDbm.has_key(key)):
snapshotDbm[key] = pickleValue
else:
# Is the file in the DB Snapshot?
if (not snapshotDbm.has_key(key)):
tmpFileObj = _encFileInfo2Obj(srvObj.getDb(),
tmpSnapshotDbm, value)
if (tmpFileObj is None):
continue
# Is the file on the disk?
complFilename = os.path.normpath(mtPt + "/" +\
tmpFileObj.getFilename())
if (os.path.exists(complFilename)):
# Add this entry in the DB Snapshot.
if (_updateSnapshot(srvObj.getCfg())):
snapshotDbm[key] = pickleValue
else:
# Remove this entry from the DB (if it is there).
_delFileEntry(srvObj.getHostId(), srvObj.getDb(),
tmpFileObj)
del tmpFileObj
else:
# We always update the DB Snapshot to ensure it is
# in-sync with the DB entry.
if (_updateSnapshot(srvObj.getCfg())):
snapshotDbm[key] = pickleValue
# Be friendly and sync the DB file every now and then
count += 1
if (count % 100) == 0:
if _updateSnapshot(srvObj.getCfg()):
snapshotDbm.sync()
checkStopJanitorThread(stopEvt)
#################################################################################################
#jagonzal: Replace looping aproach to avoid exceptions coming from the next() method underneath
# when iterating at the end of the table that are prone to corrupt the hash table object
#try:
# key, pickleValue = tmpSnapshotDbm.next()
#except:
# key = None
#################################################################################################
logger.debug("Checked DB Snapshot against DB - %s", diskId)
# End-Loop: Check DB Snapshot against DB. ###########################
if (_updateSnapshot(srvObj.getCfg())):
snapshotDbm.sync()
finally:
if snapshotDbm:
snapshotDbm.close()
if tmpSnapshotDbm:
tmpSnapshotDbm.close()
# Check if lost files found.
logger.debug("Check if there are Lost Files ...")
noOfLostFiles = lostFileRefsDbm.getCount()
if (noOfLostFiles):
statRep = os.path.normpath(tmpDir + "/" + ngasId +\
"_LOST_FILES_NOTIF_EMAIL.txt")
fo = open(statRep, "w")
timeStamp = toiso8601()
tmpFormat = "JANITOR THREAD - LOST FILES DETECTED:\n\n" +\
"==Summary:\n\n" +\
"Date: %s\n" +\
"NGAS Host ID: %s\n" +\
"Lost Files: %d\n\n" +\
"==File List:\n\n"
fo.write(tmpFormat % (timeStamp, srvObj.getHostId(), noOfLostFiles))
tmpFormat = "%-32s %-32s %-12s %-80s\n"
fo.write(tmpFormat %
("Disk ID", "File ID", "File Version", "Expected Path"))
fo.write(tmpFormat % (32 * "-", 32 * "-", 12 * "-", 80 * "-"))
# Loop over the files an generate the report.
lostFileRefsDbm.initKeyPtr()
while (1):
key, fileInfoObj = lostFileRefsDbm.getNext()
if (not key): break
diskId = fileInfoObj.getDiskId()
fileId = fileInfoObj.getFileId()
fileVersion = fileInfoObj.getFileVersion()
filename = fileInfoObj.getTag()
fo.write(tmpFormat % (diskId, fileId, fileVersion, filename))
fo.write("\n\n==END\n")
fo.close()
ngamsNotification.notify(srvObj.getHostId(), srvObj.getCfg(),
NGAMS_NOTIF_DATA_CHECK,
"LOST FILE(S) DETECTED", statRep, [], 1,
NGAMS_TEXT_MT, "JANITOR_THREAD_LOST_FILES", 1)
rmFile(statRep)
logger.debug("Number of lost files found: %d", noOfLostFiles)
# Clean up.
del lostFileRefsDbm
rmFile(lostFileRefsDbmName + "*")
def handleCmd(srvObj, reqPropsObj, httpRef):
"""
Handle Command QUERY to query the DB system used.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
T = TRACE()
# Get command parameters.
if not 'query' in reqPropsObj:
raise Exception("No query specified. Valid queries are: %s" %
(queries.keys(), ))
query = reqPropsObj.getHttpPar("query").lower()
if query not in queries.keys():
raise Exception("Invalid query specified. Valid queries are: %s" %
(queries.keys(), ))
out_format = None
if 'format' in reqPropsObj:
out_format = reqPropsObj["format"]
cursorId = None
if (reqPropsObj.hasHttpPar("cursor_id")):
cursorId = reqPropsObj.getHttpPar("cursor_id")
fetch = None
if (reqPropsObj.hasHttpPar("fetch")):
fetch = int(reqPropsObj.getHttpPar("fetch"))
# Select the SQL statement + pars to execute
colnames, sql = queries[query]
args = ()
if query in ('subscribers_like', 'files_like', 'files_location',
'lastver_location'):
param = '%'
if (reqPropsObj.hasHttpPar("like")):
param = reqPropsObj.getHttpPar("like")
args = (param, )
elif query == 'files_between':
param1 = param2 = ''
if 'start' in reqPropsObj:
param1 = reqPropsObj["start"]
if 'end' in reqPropsObj:
param2 = reqPropsObj["end"]
if param1 and param2:
args = (param1, param2)
elif param1:
sql = 'select * from ngas_files where ingestion_date >= {0}'
args = (param1, )
else:
sql = queries['files_list']
# Execute the query.
if not cursorId:
# TODO: Make possible to return an XML document
# TODO: Potential problem with very large result sets.
# Implement streaming result directly3
# rtobar, 2018 Feb: for the streaming result functionality to work
# correctly we need to support sending content with
# chunked transfer encoding first. This is not
# difficult to add on the server side, but needs
# to be also added on the client side (which I suppose
# means all languages, not only python, if we really
# need to maintain those).
# We already support proper cursors at the database
# access level, so it's not difficult to change
# the query below to use a cursor and work as a
# generator instead of returning the full list of
# results in one go.
res = srvObj.getDb().query2(sql, args=args)
if out_format in ("list", 'text'):
finalRes = formatAsList(res, colnames)
mimeType = NGAMS_TEXT_MT
elif out_format == "pickle":
finalRes = cPickle.dumps([res])
mimeType = NGAMS_PYTHON_PICKLE_MT
elif out_format == "json":
results = [{colname: val
for colname, val in zip(colnames, row)} for row in res]
finalRes = six.b(json.dumps(results, default=encode_decimal))
mimeType = NGAMS_JSON_MT
else:
finalRes = six.b(str(list(res)))
mimeType = NGAMS_PYTHON_LIST_MT
# Return the data and good bye.
httpRef.send_data(finalRes, mimeType)
return
# TODO:
#
#
#
# The rest seems to be very old functionality, which we probably
# want to drop at some point. I'm still keeping it here for the time being
# to be a good citizen and not break any (very potential) user.
#
#
#
elif (fetch):
cursorDbmFilename = genCursorDbmName(srvObj.getCfg().\
getRootDirectory(), cursorId)
if (not os.path.exists(cursorDbmFilename)):
logger.error("Illegal Cursor ID: %s or cursor expired", cursorId)
return []
try:
cursorDbm = ngamsDbm.ngamsDbm(cursorDbmFilename, writePerm=1)
count = 0
resSet = []
cursorIdx = cursorDbm.get(CURSOR_IDX)
while (count < fetch):
cursorIdx += 1
if (cursorDbm.hasKey(str(cursorIdx))):
res = cursorDbm.get(str(cursorIdx))
resSet.append(res)
else:
# -- no more results to return.
break
count += 1
cursorDbm.add(CURSOR_IDX, cursorIdx)
cursorDbm.sync()
del cursorDbm
# If all entries have been fetched, we delete the cursor DBM.
if (count < fetch):
rmFile(cursorDbmFilename + "*")
# Return the data.
# TODO: Make it possible to return ASCII List + XML.
httpRef.send_data(str(resSet), NGAMS_PYTHON_LIST_MT)
except Exception as e:
msg = "Error fetching from cursor with ID: %s. Error: %s"
raise Exception(msg % (cursorId, str(e)))
elif (query and cursorId):
logger.debug("Creating new cursor with ID: %s, query: %s", cursorId,
query)
cursorDbmFilename = genCursorDbmName(srvObj.getCfg().\
getRootDirectory(), cursorId)
cursorDbm = ngamsDbm.ngamsDbm(cursorDbmFilename, writePerm=1)
# Make the query in a cursor and dump the results into the DBM.
curObj = srvObj.getDb().dbCursor(query, args=args)
with curObj:
for res in curObj.fetch(1000):
cursorDbm.addIncKey(res)
cursorDbm.add(CURSOR_IDX, 0)
cursorDbm.sync()
del cursorDbm
# TODO: In this case no reply is generated??
else:
msg = "Error illegal combination of parameters. Correct syntax is: " +\
"QUERY?query=<Query>[&out_format=<Format (list)>] or " +\
"QUERY?query=<Query>&cursor_id=<ID> followed by N calls to " +\
"QUERY?cursor_id=<ID>&fetch=<Number of Elements>"
raise Exception(msg)
if (count < fetch):
rmFile(cursorDbmFilename + "*")
# Return the data.
# TODO: Make it possible to return ASCII List + XML.
httpRef.send_data(str(resSet), NGAMS_PYTHON_LIST_MT)
except Exception, e:
msg = "Error fetching from cursor with ID: %s. Error: %s"
raise Exception, msg % (cursorId, str(e))
elif (query and cursorId):
logger.debug("Creating new cursor with ID: %s, query: %s", cursorId,
query)
cursorDbmFilename = genCursorDbmName(srvObj.getCfg().\
getRootDirectory(), cursorId)
cursorDbm = ngamsDbm.ngamsDbm(cursorDbmFilename, writePerm=1)
# Make the query in a cursor and dump the results into the DBM.
curObj = srvObj.getDb().dbCursor(query, args=args)
with curObj:
for res in curObj.fetch(1000):
cursorDbm.addIncKey(res)
cursorDbm.add(CURSOR_IDX, 0)
cursorDbm.sync()
del cursorDbm
# TODO: In this case no reply is generated??
else:
msg = "Error illegal combination of parameters. Correct syntax is: " +\
"QUERY?query=<Query>[&out_format=<Format (list)>] or " +\
"QUERY?query=<Query>&cursor_id=<ID> followed by N calls to " +\
"QUERY?cursor_id=<ID>&fetch=<Number of Elements>"
