def get_disks_to_check(srvObj):
# Get mounted disks
slotIdList = srvObj.getDb().getSlotIdsMountedDisks(srvObj.getHostId())
disks_to_check = srvObj.getDb().\
getDiskInfoForSlotsAndHost(srvObj.getHostId(), slotIdList)
# Turn from simply SQL results into objects indexed by disk_id
disks_to_check = [
ngamsDiskInfo.ngamsDiskInfo().unpackSqlResult(x)
for x in disks_to_check
]
disks_to_check = {x.getDiskId(): x for x in disks_to_check}
# Filter out those that don't need a check
now = time.time()
check_period = isoTime2Secs(srvObj.getCfg().getDataCheckMinCycle())
def needs_check(x):
last_check = x.getLastCheck() or 0
return check_period + last_check < now
disks_to_check = {
k: v
for k, v in disks_to_check.items() if needs_check(v)
}
logger.info("Will check %d disks that are mounted in this system",
len(disks_to_check))
return disks_to_check
def test_get_file_info_list_with_wildcards(self):
"""Double-check that wildcards in fileId work"""
disk_info = ngamsDiskInfo.ngamsDiskInfo()
disk_info.setDiskId('disk-id')
disk_info.write(self.db)
file_info = ngamsFileInfo.ngamsFileInfo()
file_info.setDiskId('disk-id')
file_info.setFileId('file-id')
file_info.write('host-id', self.db, genSnapshot=0)
res = list(self.db.getFileInfoList('disk-id', fileId="*"))
self.assertEqual(1, len(res))
def get_target_volume(ngams_server):
"""
Get a random target volume with availability
:param ngams_server: Reference to NG/AMS server class object (ngamsServer)
:return: Target volume object or None (ngamsDiskInfo | None)
"""
sql = "select %s from ngas_disks nd " \
"where completed=0 AND host_id={0} order by available_mb desc" % ngamsDbCore.getNgasDisksCols()
result = ngams_server.getDb().query2(sql, args=(ngams_server.getHostId(),))
if not result:
return None
else:
return ngamsDiskInfo.ngamsDiskInfo().unpackSqlResult(result[0])
def getMountedDiskInfo(srvObj, mountPoint):
diskinfo = _cached_disk_info.get((srvObj.getHostId(), mountPoint), None)
if diskinfo == None:
sqlQuery = "SELECT %s FROM ngas_disks nd WHERE mount_point = {0} and host_id = {1}" % ngamsDbCore.getNgasDisksCols(
)
res = srvObj.getDb().query2(sqlQuery,
args=(mountPoint, srvObj.getHostId()))
if not res:
diskinfo = None
else:
diskinfo = ngamsDiskInfo.ngamsDiskInfo().unpackSqlResult(res[0])
_cached_disk_info[(srvObj.getHostId(), mountPoint)] = diskinfo
return diskinfo
def getTargetVolume(srvObj):
"""
Get the volume with most space available
srvObj: Reference to NG/AMS server class object (ngamsServer).
Returns: Target volume object or None (ngamsDiskInfo | None).
"""
sqlQuery = GET_AVAIL_VOLS_QUERY % ngamsDbCore.getNgasDisksCols()
res = srvObj.getDb().query2(sqlQuery, args=(srvObj.getHostId(), ))
if not res:
return None
else:
return ngamsDiskInfo.ngamsDiskInfo().unpackSqlResult(res[0])
def getTargetVolume(srvObj):
"""
Get a random target volume with availability.
srvObj: Reference to NG/AMS server class object (ngamsServer).
Returns: Target volume object or None (ngamsDiskInfo | None).
"""
sqlQuery = "SELECT %s FROM ngas_disks nd WHERE completed=0 AND " + \
"host_id={0} order by available_mb desc" % ngamsDbCore.getNgasDisksCols()
res = srvObj.getDb().query2(sqlQuery, args=(srvObj.getHostId(), ))
if not res:
return None
else:
return ngamsDiskInfo.ngamsDiskInfo().unpackSqlResult(res[0])
def get_mounted_disk_info(ngams_server, mount_point):
disk_info = _cached_disk_info.get((ngams_server.getHostId(), mount_point),
None)
if disk_info is None:
sql = "select %s from ngas_disks nd where mount_point = {0} and host_id = {1}"\
% ngamsDbCore.getNgasDisksCols()
result = ngams_server.getDb().query2(sql,
args=(mount_point,
ngams_server.getHostId()))
if not result:
disk_info = None
else:
disk_info = ngamsDiskInfo.ngamsDiskInfo().unpackSqlResult(
result[0])
_cached_disk_info[(ngams_server.getHostId(),
mount_point)] = disk_info
return disk_info
def _cloneExplicit(srvObj, reqPropsObj, diskId, fileId, fileVersion,
targetDiskId):
"""
Execute CLONE Command, where the source Disk ID, File ID and File Version
are specified. Is much faster than a normal CLONE Command when an explicit
file is specified.
srvObj: Reference to instance of Server Object (ngamsServer).
fileInfoObj: File info object with info about the file
diskId: ID of disk hosting the file to be cloned (string).
fileId: ID of file to clone (string).
fileVersion: Version of file to clone (integer).
targetDiskId: ID of target disk (string).
Returns: Void.
"""
T = TRACE(1)
# Resolve the location of the file to clone.
location, hostId, ipAddress, portNo, mountPoint, filename,\
fileVersion, mimeType =\
ngamsFileUtils.quickFileLocate(srvObj, reqPropsObj, fileId,
diskId=diskId,
fileVersion=fileVersion)
# Read also the entire file info (unfortunately).
srcFileInfo = ngamsFileInfo.ngamsFileInfo().read(srvObj.getHostId(),
srvObj.getDb(), fileId,
fileVersion, diskId)
# Determine target disk.
if (targetDiskId == ""):
# Try to find a disk not hosting already a file with that
# ID + version.
diskExemptList = [diskId]
while (1):
trgDiskInfo = ngamsDiskUtils.\
findTargetDisk(srvObj.getHostId(),
srvObj.getDb(), srvObj.getCfg(),
mimeType, 1, diskExemptList)
# Check if a file with that ID + version is already
# stored on the selected Target Disk.
if (srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(), fileId,
fileVersion)):
# This file is already stored on the given disk.
# Add to the exempt list.
diskExemptList.append(trgDiskInfo.getDiskId())
else:
# OK, this disk should be OK, stop looking for a
# suitable Target Disk.
break
else:
trgDiskInfo = ngamsDiskInfo.ngamsDiskInfo().\
read(srvObj.getDb(), targetDiskId)
slotId = trgDiskInfo.getSlotId()
storageSetId = srvObj.getCfg().getStorageSetFromSlotId(slotId).\
getStorageSetId()
trgDiskInfo.setStorageSetId(storageSetId)
# Don't accept to clone onto the same disk (this would meann overwriting).
if (trgDiskInfo.getDiskId() == diskId):
err = "Source and target files are identical"
msg = "Failed in cloning file with ID: " + fileId +\
"/Version: " + str(fileVersion) +\
" on disk with ID: " + diskId +\
" on host: " + hostId + ". Reason: " + err
raise Exception, msg
# Receive the file into the staging filename.
tmpReqPropsObj = ngamsReqProps.ngamsReqProps()
tmpReqPropsObj.setMimeType(mimeType)
stagingFilename = ngamsHighLevelLib.genStagingFilename(
srvObj.getCfg(), tmpReqPropsObj, trgDiskInfo, fileId)
try:
quickLocation = False
if (reqPropsObj.hasHttpPar("quick")):
quickLocation = int(reqPropsObj.getHttpPar("quick"))
# Receive the data into the Staging File using the urllib.
if (srvObj.getHostId() != hostId):
# Example: http://host:7777/RETRIEVE?disk_id=%s&"
# file_id=id&file_version=1
fileUrl = "http://%s:%s/RETRIEVE?disk_id=%s&file_id=%s&" +\
"file_version=%s"
fileUrl = fileUrl % (ipAddress, str(portNo), diskId, fileId,
str(fileVersion))
# If CLONE?quick specified, we try to retrieve the file via the
# RETRIEVE?quick_location method.
quickFileUrl = fileUrl
if (reqPropsObj.hasHttpPar("quick")):
if (int(reqPropsObj.getHttpPar("quick"))):
quickFileUrl = fileUrl + "&quick_location=1"
# Check if host is suspended, if yes, wake it up.
if (srvObj.getDb().getSrvSuspended(hostId)):
logger.debug("Clone Request - Waking up suspended " +\
"NGAS Host: %s", hostId)
ngamsSrvUtils.wakeUpHost(srvObj, hostId)
else:
# TODO: a time-bomb waiting to explode....
fileUrl = "file:" + mtPt + "/" + filename
logger.debug("Receiving file via URI: %s into staging filename: %s",
fileUrl, stagingFilename)
# We try up to 5 times to retrieve the file in case a problem is
# encountered during cloning.
for attempt in range(5):
try:
if (attempt == 0):
filename, headers = urllib.urlretrieve(
quickFileUrl, stagingFilename)
else:
filename, headers = urllib.urlretrieve(
fileUrl, stagingFilename)
_checkFile(srvObj, srcFileInfo, stagingFilename, headers, True)
# If we get to this point the transfer was (probably) OK.
break
except Exception, e:
rmFile(stagingFilename)
errMsg = "Problem occurred while cloning file "+\
"via URL: " + fileUrl + " - Error: " + str(e)
if (attempt < 4):
errMsg += " - Retrying in 5s ..."
logger.error(errMsg)
time.sleep(0.5)
else:
raise Exception, errMsg
# We simply copy the file into the same destination as the
# source file (but on another disk).
targPathName = os.path.dirname(srcFileInfo.getFilename())
targFilename = os.path.basename(srcFileInfo.getFilename())
complTargPath = os.path.normpath(trgDiskInfo.getMountPoint() +\
"/" + targPathName)
checkCreatePath(complTargPath)
complFilename = os.path.normpath(complTargPath + "/" + targFilename)
mvTime = mvFile(stagingFilename, complFilename)
ngamsLib.makeFileReadOnly(complFilename)
# Update status for new file in the DB.
newFileInfo = srcFileInfo.clone().setDiskId(trgDiskInfo.getDiskId()).\
setCreationDate(getFileCreationTime(complFilename))
fileExists = srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(), fileId,
fileVersion)
newFileInfo.write(srvObj.getHostId(), srvObj.getDb())
# Update status for the Target Disk in DB + check if the disk is
# completed.
if (fileExists): mvTime = 0
dummyDapiStatObj = ngamsDapiStatus.ngamsDapiStatus().\
setDiskId(trgDiskInfo.getDiskId()).\
setFileExists(fileExists).\
setFileSize(srcFileInfo.getFileSize()).\
setIoTime(mvTime)
ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), dummyDapiStatObj)
ngamsArchiveUtils.checkDiskSpace(srvObj, trgDiskInfo.getDiskId())
# If running as a cache archive, update the Cache New Files DBM
# with the information about the new file.
if (srvObj.getCachingActive()):
ngamsCacheControlThread.addEntryNewFilesDbm(
srvObj, diskId, fileId, fileVersion, filename)
# Generate a confirmation log entry.
msg = genLog("NGAMS_INFO_FILE_CLONED",
[fileId, fileVersion, diskId, hostId])
logger.info(msg, extra={'to_syslog': True})
def _cloneExec(srvObj, cloneListDbmName, tmpFilePat, targetDiskId,
reqPropsObj):
"""
See documentation of ngamsCloneCmd._cloneThread(). This function is
merely implemented in order to encapsulate the whole process to be able
to clean up properly when the processing is terminated.
"""
T = TRACE(1)
cloneListDbm = cloneStatusDbm = None
emailNotif = 0
checkChecksum = 1
if (reqPropsObj):
if (reqPropsObj.hasHttpPar("notif_email")): emailNotif = 1
if (reqPropsObj.hasHttpPar("check")):
checkChecksum = int(reqPropsObj.getHttpPar("check"))
# Open clone list DB.
cloneListDbm = ngamsDbm.ngamsDbm(cloneListDbmName)
# We have to get the port numbers of the hosts where the files to be
# cloned are stored.
hostInfoDic = {}
cloneListDbm.initKeyPtr()
while (1):
key, fileInfo = cloneListDbm.getNext()
if (not key): break
hostInfoDic[fileInfo[1]] = -1
hostInfoDic = ngamsHighLevelLib.resolveHostAddress(srvObj.getHostId(),
srvObj.getDb(),
srvObj.getCfg(),
hostInfoDic.keys())
# The cloning loop. Loop over the list of files to clone and generate
# a report with the result.
if (emailNotif):
cloneStatusDbmName = tmpFilePat + "_CLONE_STATUS_DB"
cloneStatusDbm = ngamsDbm.ngamsDbm(cloneStatusDbmName,
cleanUpOnDestr=0,
writePerm=1)
successCloneCount = 0
failedCloneCount = 0
abortCloneLoop = 0
timeAccu = 0.0
key = 0
while (1):
clone_start = time.time()
if (not cloneListDbm.hasKey(str(key))): break
fileInfo = cloneListDbm.get(str(key))
key += 1
# Check if we have permission to run. Otherwise, stop.
if (not srvObj.getThreadRunPermission()): break
fio = fileInfo[0]
mtPt = fileInfo[2]
if (emailNotif):
tmpFileList = ngamsFileList.\
ngamsFileList("FILE_CLONE_STATUS",
"File: " + fio.getFileId() + "/" +\
fio.getDiskId() + "/" +\
str(fio.getFileVersion()))
hostId = fileInfo[1]
text = "Cloning file - File ID: %s/%d, on disk " +\
"with ID: %s on host: %s"
logger.debug(text, fio.getFileId(), fio.getFileVersion(),
fio.getDiskId(), hostId)
# We generate a local Staging File and archive this.
stagingFilename = ""
try:
# Check if file is marked as bad.
if (fio.getFileStatus()[0] == "1"):
errMsg = "File marked as bad - skipping!"
raise Exception, errMsg
if (targetDiskId == ""):
# Try to find a disk not hosting already a file with that
# ID + version.
diskExemptList = [fio.getDiskId()]
while (1):
trgDiskInfo = ngamsDiskUtils.\
findTargetDisk(srvObj.getHostId(),
srvObj.getDb(),
srvObj.getCfg(),
fio.getFormat(),
1, diskExemptList)
# Check if a file with that ID + version is already
# stored on the selected Target Disk.
if (srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(),
fio.getFileId(),
fio.getFileVersion())):
# This file is already stored on the given disk.
# Add to the exempt list.
diskExemptList.append(trgDiskInfo.getDiskId())
else:
# OK, this disk should be OK, stop looking for a
# suitable Target Disk.
break
else:
try:
trgDiskInfo = ngamsDiskInfo.ngamsDiskInfo().\
read(srvObj.getDb(), targetDiskId)
slotId = trgDiskInfo.getSlotId()
storageSetId = srvObj.getCfg().\
getStorageSetFromSlotId(slotId).\
getStorageSetId()
trgDiskInfo.setStorageSetId(storageSetId)
except Exception:
abortCloneLoop = 1
raise
# We don't accept to clone onto the same disk (this would mean
# overwriting).
if (trgDiskInfo.getDiskId() == fio.getDiskId()):
err = "Source and target files are identical"
msg = "Failed in cloning file with ID: " + fio.getFileId() +\
"/Version: " + str(fio.getFileVersion()) +\
" on disk with ID: " + fio.getDiskId() +\
" on host: " + hostId + ". Reason: " + err
logger.warning(msg)
if (emailNotif):
tmpFileList.setStatus(NGAMS_FAILURE + ": " + err)
tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE"))
cloneStatusDbm.addIncKey(tmpFileList)
failedCloneCount += 1
continue
tmpReqPropsObj = ngamsReqProps.ngamsReqProps()
tmpReqPropsObj.setMimeType(fio.getFormat())
stagingFilename = ngamsHighLevelLib.\
genStagingFilename(srvObj.getCfg(),
tmpReqPropsObj,
trgDiskInfo, fio.getFileId())
# Receive the data into the Staging File using the urllib.
if (srvObj.getHostId() != hostId):
# Example: http://host:7777/RETRIEVE?file_id=id&file_version=1
ipAddress = hostInfoDic[hostId].getIpAddress()
portNo = hostInfoDic[hostId].getSrvPort()
fileUrl = "http://" + ipAddress + ":" + str(portNo) +\
"/RETRIEVE?" + "file_id=" + fio.getFileId() +\
"&file_version=" + str(fio.getFileVersion())
# If a specific Disk ID for the source file is given, append
# this.
if (fio.getDiskId()):
fileUrl += "&disk_id=%s" % fio.getDiskId()
# Check if host is suspended, if yes, wake it up.
if (srvObj.getDb().getSrvSuspended(hostId)):
logger.debug("Clone Request - Waking up suspended " +\
"NGAS Host: %s", hostId)
ngamsSrvUtils.wakeUpHost(srvObj, hostId)
else:
fileUrl = "file:" + mtPt + "/" + fio.getFilename()
logger.debug(
"Receiving file via URI: %s into staging filename: %s",
fileUrl, stagingFilename)
# We try up to 5 times to retrieve the file in case a problem is
# encountered during cloning.
for attempt in range(5):
try:
filename, headers = urllib.urlretrieve(
fileUrl, stagingFilename)
_checkFile(srvObj, fio, stagingFilename, headers,
checkChecksum)
# If we get to this point the transfer was (probably) OK.
break
except Exception, e:
rmFile(stagingFilename)
errMsg = "Problem occurred while cloning file "+\
"via URL: " + fileUrl + " - Error: " + str(e)
if (attempt < 4):
errMsg += " - Retrying in 5s ..."
logger.error(errMsg)
time.sleep(5)
else:
raise Exception, errMsg
# We simply copy the file into the same destination as the
# source file (but on another disk).
targPathName = os.path.dirname(fio.getFilename())
targFilename = os.path.basename(fio.getFilename())
complTargPath = os.path.normpath(trgDiskInfo.getMountPoint() +\
"/" + targPathName)
checkCreatePath(complTargPath)
complFilename = os.path.normpath(complTargPath + "/" +
targFilename)
mvTime = mvFile(stagingFilename, complFilename)
ngamsLib.makeFileReadOnly(complFilename)
# Update status for new file in the DB.
newFileInfo = fio.clone().setDiskId(trgDiskInfo.getDiskId()).\
setCreationDate(getFileCreationTime(complFilename))
fileExists = srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(),
fio.getFileId(),
fio.getFileVersion())
newFileInfo.write(srvObj.getHostId(), srvObj.getDb())
# Update status for the Target Disk in DB + check if the disk is
# completed.
if (fileExists): mvTime = 0
dummyDapiStatObj = ngamsDapiStatus.ngamsDapiStatus().\
setDiskId(trgDiskInfo.getDiskId()).\
setFileExists(fileExists).\
setFileSize(fio.getFileSize()).setIoTime(mvTime)
ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), dummyDapiStatObj)
ngamsArchiveUtils.checkDiskSpace(srvObj, trgDiskInfo.getDiskId())
# Update the clone file status list.
if (emailNotif):
tmpFileList.setStatus(NGAMS_SUCCESS)
tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE"))
tmpFileList.addFileInfoObj(newFileInfo.setTag("TARGET_FILE"))
cloneStatusDbm.addIncKey(tmpFileList)
successCloneCount += 1
# If running as a cache archive, update the Cache New Files DBM
# with the information about the new file.
if (srvObj.getCachingActive()):
diskId = trgDiskInfo.getDiskId()
fileId = fio.getFileId()
fileVer = fio.getFileVersion()
filename = fio.getFilename()
ngamsCacheControlThread.addEntryNewFilesDbm(
srvObj, diskId, fileId, fileVer, filename)
# Generate a confirmation log entry.
cloneTime = time.time() - clone_start
timeAccu += cloneTime
msg = genLog("NGAMS_INFO_FILE_CLONED", [
fio.getFileId(),
fio.getFileVersion(),
fio.getDiskId(), hostId
])
msg = msg + ". Time: %.3fs. Total time: %.3fs." %\
(cloneTime, timeAccu)
logger.info(msg, extra={'to_syslog': True})
except Exception, e:
cloneTime = time.time() - clone_start
timeAccu += cloneTime
errMsg = genLog("NGAMS_ER_FILE_CLONE_FAILED", [
fio.getFileId(),
fio.getFileVersion(),
fio.getDiskId(), hostId,
str(e)
])
if (abortCloneLoop):
logger.error(errMsg, extra={'to_syslog': True})
thread.exit()
else:
logger.warning(errMsg)
if (emailNotif):
tmpFileList.setStatus(NGAMS_FAILURE + ": Error: " + errMsg)
tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE"))
cloneStatusDbm.addIncKey(tmpFileList)
failedCloneCount += 1
# Delete Staging File if already created.
if ((stagingFilename != "") and (os.path.exists(stagingFilename))):
rmFile(stagingFilename)
def cloneCheckDiskSpace(srvObj, cloneListDbmName, tmpFilePat, targetDiskId=""):
"""
Check if there is enough disk space available on this NGAS host for
carrying out the Clone Request.
srvObj: Reference to instance of Server Object (ngamsServer).
cloneListDbmName: Name of DBM containing the information about
the files to be cloned. This DB has an index number
as key pointing to pickled information about each file.
This pickled information is
[<File Info Object>, <Host ID>, <Mount Point>]
(string)
tmpFilePat: File pattern to be used for generating temporary
files (string).
targetDiskId: ID of disk to where the files cloned should be
written (string).
Returns: Void.
"""
T = TRACE()
# Make a dictionary with the available amount of space per disk.
logger.debug("Generating dictionary with available space per disk")
availDiskSpaceDic = {}
mtRootDir = srvObj.getCfg().getRootDirectory()
if (targetDiskId):
tmpDiskInfoObjList = [ngamsDiskInfo.ngamsDiskInfo().\
read(srvObj.getDb(), targetDiskId)]
else:
tmpDiskInfoObjList = ngamsDiskUtils.\
getDiskInfoForMountedDisks(srvObj.getDb(),
srvObj.getHostId(),
mtRootDir)
for diskInfoObj in tmpDiskInfoObjList:
mtPt = diskInfoObj.getMountPoint()
diskId = diskInfoObj.getDiskId()
availDiskSpaceDic[diskId] = getDiskSpaceAvail(mtPt, format="B")
# Now simulate the execution of the clone job by going sequentially
# through the files selected for cloning and subtract the required
# disk space from the available amount for each disk.
#
# When a disk reaches the threshold for considering the disk as
# completed, it is removed from the list of available disks.
cloneListDbm = ngamsDbm.ngamsDbm(cloneListDbmName)
# This dictionary contains a list of files that have been simulated
# successfully cloned. Since they are not updated in the DB, we keep
# track of this locally.
cloneSucDbmName = tmpFilePat + "_CLONE_SUCCESS_DB"
rmFile(cloneSucDbmName + "*")
cloneSucDbm = ngamsDbm.ngamsDbm(cloneSucDbmName,
cleanUpOnDestr=1,
writePerm=1)
# This is used to keep track of the files that could not be cloned
# due to lack of space.
cloneFailDbmName = tmpFilePat + "_CLONE_FAILED_DB"
rmFile(cloneFailDbmName + "*")
cloneFailDbm = ngamsDbm.ngamsDbm(cloneFailDbmName,
cleanUpOnDestr=1,
writePerm=1)
# This is used to simulate disk completion. If a specific target
# disk is defined, all other disks should be ignored (=we set them to
# completed).
cloneSimDiskCompl = []
if (targetDiskId):
tmpDiskList = ngamsDiskUtils.\
getDiskInfoForMountedDisks(srvObj.getDb(), srvObj.getHostId(),
mtRootDir)
for idx in range(len(tmpDiskList)):
if (tmpDiskList[idx].getDiskId() != targetDiskId):
cloneSimDiskCompl.append(tmpDiskList[idx].getDiskId())
# Carry out the simulated clone process.
ngamsDiskUtils.findTargetDiskResetCache()
key = 0
while (1):
if (not cloneListDbm.hasKey(str(key))): break
fileInfo = cloneListDbm.get(str(key))
key += 1
fio = fileInfo[0]
hostName = fileInfo[1]
text = "Simulating cloning of file - File ID: %s/%d, on disk " +\
"with ID: %s on host: %s"
logger.debug(text, fio.getFileId(), fio.getFileVersion(),
fio.getDiskId(), hostName)
diskExemptList = cloneSimDiskCompl + [fio.getDiskId()]
trgDiskInfo = None
while (1):
try:
trgDiskInfo = ngamsDiskUtils.\
findTargetDisk(srvObj.getHostId(),
srvObj.getDb(), srvObj.getCfg(),
fio.getFormat(), 0,
diskExemptList, 1)
except Exception, e:
if (str(e).find("NGAMS_AL_NO_STO_SETS") != -1):
# No more candidate Target Disks for this type
# of data - this file cannot be cloned.
cloneFailDbm.addIncKey(fio)
break
# Check if a file with that ID + version is already
# stored on the selected Target Disk.
fileInDb = srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(),
fio.getFileId(),
fio.getFileVersion())
fileKey = ngamsLib.genFileKey(trgDiskInfo.getDiskId(),
fio.getFileId(),
fio.getFileVersion())
fileSimCloned = cloneSucDbm.hasKey(fileKey)
if (fileInDb or fileSimCloned):
# This file is already stored on the given disk.
# Add to the exempt list.
diskExemptList.append(trgDiskInfo.getDiskId())
else:
# OK, this disk should be OK, stop looking for a
# suitable Target Disk.
logger.debug("Found suitable disk with ID: %s/Slot ID: %s",
trgDiskInfo.getDiskId(), trgDiskInfo.getSlotId())
cloneSucDbm.add(fileKey, "")
break
# We now subtract the size of the file from the available amount of
# disk space for the selected Target Disk. When the amount of available
# space goes below the threshold defined for this NG/AMS system that
# disk is considered to be completed.
if (trgDiskInfo):
diskId = trgDiskInfo.getDiskId()
availDiskSpaceDic[diskId] -= float(fio.getFileSize())
if ((availDiskSpaceDic[diskId] / 1048576.0) < \
srvObj.getCfg().getFreeSpaceDiskChangeMb()):
cloneSimDiskCompl.append(diskId)
os.system("rm -f %s*" % tmpFile1)
except Exception, e:
os.system("rm -f %s*" % tmpFile1)
raise e
else:
arcFile, dpId, dateDir = ngamsFitsPlugIn.getDpIdInfo(baseFile)
uncomprSize = getFileSize(baseFile)
insId = arcFile.split(".")[0]
isoTime = arcFile[(arcFile.find(".") + 1):]
baseTimeMjd = PccUtTime.TimeStamp(isoTime).getMjd()
# Generate 'static' file info for this run.
ngasDiskInfo = os.path.normpath("%s/NgasDiskInfo" %\
testParDic["MOUNTPOINT"])
ngasDiskInfoDoc = open(ngasDiskInfo).read()
diskInfoObj = ngamsDiskInfo.ngamsDiskInfo().\
unpackXmlDoc(ngasDiskInfoDoc, ignoreVarDiskPars=1)
diskId = diskInfoObj.getDiskId()
fileVer = 1
if (ext == "Z"):
format = "application/x-cfits"
compression = "compress -f"
elif (ext == "gz"):
format = "application/x-gfits"
compression = "gzip"
else:
format = "image/x-fits"
compression = "NONE"
archFileSize = getFileSize(baseFile)
ignore = 0
checksumPlugIn = "ngamsGenCrc32"
exec "import " + checksumPlugIn
def _cloneExec(srvObj,
cloneListDbmName,
tmpFilePat,
targetDiskId,
reqPropsObj):
"""
See documentation of ngamsCloneCmd._cloneThread(). This function is
merely implemented in order to encapsulate the whole process to be able
to clean up properly when the processing is terminated.
"""
cloneStatusDbm = None
emailNotif = 0
checkChecksum = 1
if (reqPropsObj):
if (reqPropsObj.hasHttpPar("notif_email")): emailNotif = 1
if (reqPropsObj.hasHttpPar("check")):
checkChecksum = int(reqPropsObj.getHttpPar("check"))
# Open clone list DB.
cloneListDbm = ngamsDbm.ngamsDbm(cloneListDbmName)
# We have to get the port numbers of the hosts where the files to be
# cloned are stored.
hostInfoDic = {}
cloneListDbm.initKeyPtr()
while (1):
key, fileInfo = cloneListDbm.getNext()
if (not key): break
hostInfoDic[fileInfo[1]] = -1
hostInfoDic = ngamsHighLevelLib.resolveHostAddress(srvObj.getHostId(),
srvObj.getDb(),
srvObj.getCfg(),
hostInfoDic.keys())
# The cloning loop. Loop over the list of files to clone and generate
# a report with the result.
if (emailNotif):
cloneStatusDbmName = tmpFilePat + "_CLONE_STATUS_DB"
cloneStatusDbm = ngamsDbm.ngamsDbm(cloneStatusDbmName,
cleanUpOnDestr = 0, writePerm = 1)
successCloneCount = 0
failedCloneCount = 0
abortCloneLoop = 0
timeAccu = 0.0
key = 0
while (1):
clone_start = time.time()
if (not cloneListDbm.hasKey(str(key))): break
fileInfo = cloneListDbm.get(str(key))
key += 1
# Check if we have permission to run. Otherwise, stop.
if (not srvObj.run_async_commands): break
fio = fileInfo[0]
mtPt = fileInfo[2]
if (emailNotif):
tmpFileList = ngamsFileList.\
ngamsFileList("FILE_CLONE_STATUS",
"File: " + fio.getFileId() + "/" +\
fio.getDiskId() + "/" +\
str(fio.getFileVersion()))
hostId = fileInfo[1]
text = "Cloning file - File ID: %s/%d, on disk " +\
"with ID: %s on host: %s"
logger.debug(text, fio.getFileId(), fio.getFileVersion(),
fio.getDiskId(), hostId)
# We generate a local Staging File and archive this.
stagingFilename = ""
try:
# Check if file is marked as bad.
if (fio.getFileStatus()[0] == "1"):
errMsg = "File marked as bad - skipping!"
raise Exception(errMsg)
if (targetDiskId == ""):
# Try to find a disk not hosting already a file with that
# ID + version.
diskExemptList = [fio.getDiskId()]
while (1):
trgDiskInfo = ngamsDiskUtils.\
findTargetDisk(srvObj.getHostId(),
srvObj.getDb(),
srvObj.getCfg(),
fio.getFormat(),
1, diskExemptList)
# Check if a file with that ID + version is already
# stored on the selected Target Disk.
if (srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(),
fio.getFileId(),
fio.getFileVersion())):
# This file is already stored on the given disk.
# Add to the exempt list.
diskExemptList.append(trgDiskInfo.getDiskId())
else:
# OK, this disk should be OK, stop looking for a
# suitable Target Disk.
break
else:
try:
trgDiskInfo = ngamsDiskInfo.ngamsDiskInfo().\
read(srvObj.getDb(), targetDiskId)
slotId = trgDiskInfo.getSlotId()
storageSetId = srvObj.getCfg().\
getStorageSetFromSlotId(slotId).\
getStorageSetId()
trgDiskInfo.setStorageSetId(storageSetId)
except Exception:
abortCloneLoop = 1
raise
# We don't accept to clone onto the same disk (this would mean
# overwriting).
if (trgDiskInfo.getDiskId() == fio.getDiskId()):
err = "Source and target files are identical"
msg = "Failed in cloning file with ID: " + fio.getFileId() +\
"/Version: " + str(fio.getFileVersion()) +\
" on disk with ID: " + fio.getDiskId() +\
" on host: " + hostId + ". Reason: " + err
logger.warning(msg)
if (emailNotif):
tmpFileList.setStatus(NGAMS_FAILURE + ": " + err)
tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE"))
cloneStatusDbm.addIncKey(tmpFileList)
failedCloneCount += 1
continue
tmpReqPropsObj = ngamsReqProps.ngamsReqProps()
tmpReqPropsObj.setMimeType(fio.getFormat())
stagingFilename = ngamsHighLevelLib.\
genStagingFilename(srvObj.getCfg(),
tmpReqPropsObj,
trgDiskInfo, fio.getFileId())
# Receive the data into the Staging File using the urllib.
if (srvObj.getHostId() != hostId):
# Example: http://host:7777/RETRIEVE?file_id=id&file_version=1
ipAddress = hostInfoDic[hostId].getIpAddress()
portNo = hostInfoDic[hostId].getSrvPort()
fileUrl = "http://" + ipAddress + ":" + str(portNo) +\
"/RETRIEVE?" + "file_id=" + fio.getFileId() +\
"&file_version=" + str(fio.getFileVersion())
# If a specific Disk ID for the source file is given, append
# this.
if (fio.getDiskId()):
fileUrl += "&disk_id=%s" % fio.getDiskId()
# Check if host is suspended, if yes, wake it up.
if (srvObj.getDb().getSrvSuspended(hostId)):
logger.debug("Clone Request - Waking up suspended " +\
"NGAS Host: %s", hostId)
ngamsSrvUtils.wakeUpHost(srvObj, hostId)
else:
fileUrl = "file:" + mtPt + "/" + fio.getFilename()
logger.debug("Receiving file via URI: %s into staging filename: %s",
fileUrl, stagingFilename)
# We try up to 5 times to retrieve the file in case a problem is
# encountered during cloning.
for attempt in range(5):
try:
filename, headers = urlrequest.urlretrieve(fileUrl, stagingFilename)
_checkFile(srvObj, fio, stagingFilename, headers,
checkChecksum)
# If we get to this point the transfer was (probably) OK.
break
except Exception as e:
rmFile(stagingFilename)
errMsg = "Problem occurred while cloning file "+\
"via URL: " + fileUrl + " - Error: " + str(e)
if (attempt < 4):
errMsg += " - Retrying in 5s ..."
logger.error(errMsg)
time.sleep(5)
else:
raise Exception(errMsg)
# We simply copy the file into the same destination as the
# source file (but on another disk).
targPathName = os.path.dirname(fio.getFilename())
targFilename = os.path.basename(fio.getFilename())
complTargPath = os.path.normpath(trgDiskInfo.getMountPoint() +\
"/" + targPathName)
checkCreatePath(complTargPath)
complFilename = os.path.normpath(complTargPath + "/"+targFilename)
mvTime = mvFile(stagingFilename, complFilename)
ngamsLib.makeFileReadOnly(complFilename)
# Update status for new file in the DB.
newFileInfo = fio.clone().setDiskId(trgDiskInfo.getDiskId()).\
setCreationDate(getFileCreationTime(complFilename))
fileExists = srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(),
fio.getFileId(),
fio.getFileVersion())
newFileInfo.write(srvObj.getHostId(), srvObj.getDb())
# Update status for the Target Disk in DB + check if the disk is
# completed.
if (fileExists): mvTime = 0
dummyDapiStatObj = ngamsDapiStatus.ngamsDapiStatus().\
setDiskId(trgDiskInfo.getDiskId()).\
setFileExists(fileExists).\
setFileSize(fio.getFileSize()).setIoTime(mvTime)
ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), dummyDapiStatObj)
ngamsArchiveUtils.checkDiskSpace(srvObj, trgDiskInfo.getDiskId())
# Update the clone file status list.
if (emailNotif):
tmpFileList.setStatus(NGAMS_SUCCESS)
tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE"))
tmpFileList.addFileInfoObj(newFileInfo.setTag("TARGET_FILE"))
cloneStatusDbm.addIncKey(tmpFileList)
successCloneCount += 1
# If running as a cache archive, update the Cache New Files DBM
# with the information about the new file.
if (srvObj.getCachingActive()):
diskId = trgDiskInfo.getDiskId()
fileId = fio.getFileId()
fileVer = fio.getFileVersion()
filename = fio.getFilename()
ngamsCacheControlThread.addEntryNewFilesDbm(srvObj, diskId,
fileId, fileVer,
filename)
# Generate a confirmation log entry.
cloneTime = time.time() - clone_start
timeAccu += cloneTime
msg = genLog("NGAMS_INFO_FILE_CLONED",
[fio.getFileId(), fio.getFileVersion(),
fio.getDiskId(), hostId])
msg = msg + ". Time: %.3fs. Total time: %.3fs." %\
(cloneTime, timeAccu)
logger.info(msg, extra={'to_syslog': True})
except Exception as e:
cloneTime = time.time() - clone_start
timeAccu += cloneTime
errMsg = genLog("NGAMS_ER_FILE_CLONE_FAILED",
[fio.getFileId(), fio.getFileVersion(),
fio.getDiskId(), hostId, str(e)])
if (abortCloneLoop):
logger.error(errMsg, extra={'to_syslog': True})
return
else:
logger.warning(errMsg)
if (emailNotif):
tmpFileList.setStatus(NGAMS_FAILURE + ": Error: " + errMsg)
tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE"))
cloneStatusDbm.addIncKey(tmpFileList)
failedCloneCount += 1
# Delete Staging File if already created.
if ((stagingFilename != "") and (os.path.exists(stagingFilename))):
rmFile(stagingFilename)
# Calculate time statistics.
if (reqPropsObj):
ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj.\
incActualCount(1), timeAccu)
# Final update of the Request Status.
if (reqPropsObj):
complPercent = (100.0 * (float(reqPropsObj.getActualCount()) /
float(reqPropsObj.getExpectedCount())))
reqPropsObj.setCompletionPercent(complPercent, 1)
reqPropsObj.setCompletionTime(1)
srvObj.updateRequestDb(reqPropsObj)
# Send Clone Report with list of files cloned to a possible
# requestor(select) of this.
totFiles = (successCloneCount + failedCloneCount)
if (emailNotif):
xmlStat = 0
# TODO: Generation of XML status report is disabled since we cannot
# handle for the moment XML documents with 1000s of elements.
if (xmlStat):
cloneStatusFileList = ngamsFileList.\
ngamsFileList("FILE_CLONING_STATUS_REPORT",
"File Cloning Status Report")
fileCount = 0
while (fileCount < cloneStatusDbm.getCount()):
tmpFileList = cloneStatusDbm.get(str(fileCount))
cloneStatusFileList.addFileListObj(tmpFileList)
# Make overall status.
cloneStatusFileList.setStatus("SUCCESS: " +\
str(successCloneCount) +\
", FAILURE: " +\
str(failedCloneCount) +\
", NOT DONE: " +\
str(len(cloneStatusFileList) -\
successCloneCount -\
failedCloneCount))
status = srvObj.genStatus(NGAMS_SUCCESS,
"CLONE command status report").\
addFileList(cloneStatusFileList)
statRep = status.genXmlDoc(0, 0, 0, 1, 0)
statRep = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, statRep)
mimeType = NGAMS_XML_MT
else:
# Generate a 'simple' ASCII report.
statRep = tmpFilePat + "_NOTIF_EMAIL.txt"
fo = open(statRep, "w")
if (reqPropsObj.hasHttpPar("disk_id")):
diskId = reqPropsObj.getHttpPar("disk_id")
else:
diskId = "-----"
if (reqPropsObj.hasHttpPar("file_id")):
fileId = reqPropsObj.getHttpPar("file_id")
else:
fileId = "-----"
if (reqPropsObj.hasHttpPar("file_version")):
fileVersion = reqPropsObj.getHttpPar("file_version")
else:
fileVersion = "-----"
tmpFormat = "CLONE STATUS REPORT:\n\n" +\
"==Summary:\n\n" +\
"Date: %s\n" +\
"NGAS Host: %s\n" +\
"Disk ID: %s\n" +\
"File ID: %s\n" +\
"File Version: %s\n" +\
"Total Number of Files: %d\n" +\
"Number of Cloned Files: %d\n" +\
"Number of Failed Files: %d\n" +\
"Total processing time (s): %.3f\n" +\
"Handling time per file (s): %.3f\n\n" +\
"==File List:\n\n"
fo.write(tmpFormat % (toiso8601(), srvObj.getHostId(), diskId, fileId,
str(fileVersion), totFiles,
successCloneCount, failedCloneCount,
timeAccu, (timeAccu / totFiles)))
tmpFormat = "%-70s %-70s %-7s\n"
fo.write(tmpFormat % ("Source File", "Target File", "Status"))
fo.write(tmpFormat % (70 * "-", 70 * "-", 7 * "-"))
key = 1
while (1):
if (not cloneStatusDbm.hasKey(str(key))): break
tmpFileList = cloneStatusDbm.get(str(key))
key += 1
srcFileObj = tmpFileList.getFileInfoObjList()[0]
srcFile = "%s/%s/%d" % (srcFileObj.getDiskId(),
srcFileObj.getFileId(),
srcFileObj.getFileVersion())
if (tmpFileList.getStatus() == NGAMS_SUCCESS):
trgFileObj = tmpFileList.getFileInfoObjList()[1]
trgFile = "%s/%s/%d" % (trgFileObj.getDiskId(),
trgFileObj.getFileId(),
trgFileObj.getFileVersion())
else:
trgFile = "-----"
fo.write(tmpFormat % (srcFile,trgFile,tmpFileList.getStatus()))
fo.write(149 * "-")
fo.write("\n\n==END\n")
fo.close()
mimeType = NGAMS_TEXT_MT
# Send out the status report.
emailAdrList = reqPropsObj.getHttpPar("notif_email").split(",")
attachmentName = "CloneStatusReport"
if (reqPropsObj.hasHttpPar("disk_id")):
attachmentName += "-" + reqPropsObj.getHttpPar("disk_id")
if (reqPropsObj.hasHttpPar("file_id")):
attachmentName += "-" + reqPropsObj.getHttpPar("file_id")
if (reqPropsObj.hasHttpPar("file_version")):
attachmentName += "-" + reqPropsObj.getHttpPar("file_version")
ngamsNotification.notify(srvObj.host_id, srvObj.cfg, NGAMS_NOTIF_INFO,
"CLONE STATUS REPORT", statRep, recList=emailAdrList, force=1,
contentType=mimeType, attachmentName=attachmentName)
del cloneStatusDbm
rmFile(cloneStatusDbmName + "*")
rmFile(statRep)
if (cloneListDbm): del cloneListDbm
rmFile(cloneListDbmName + "*")
logger.info("_cloneExec(). Total time: %.3fs. Average time per file: %.3fs.",
timeAccu, (timeAccu / totFiles))
def check_cloning(source_disk_id, auto_clone):
"""
Function to check if all files registered for a given disk, have been cloned onto disks in the system where this
tool is executed.
:param source_disk_id: ID of the cloned disk (string)
:param auto_clone: If a file is not cloned will be automatically cloned by the tool (integer/0|1)
:return: Status message (string)
"""
if auto_clone:
port_num = ngasUtilsLib.get_parameter_ngas_resource_file("NgasPort")
ngams_client = ngamsPClient.ngamsPClient(getHostName(), port_num)
else:
ngams_client = None
check_report_main = 50 * "=" + "\n"
check_report_main += "Clone Verification Report - Disk: {:s}\n\n".format(
source_disk_id)
check_report = ""
connection = ngasUtilsLib.get_db_connection()
# Get information about files on the Source Disk
disk_info = ngamsDiskInfo.ngamsDiskInfo()
disk_info = disk_info.read(connection, source_disk_id)
if not disk_info:
raise Exception("Specified Disk ID: {:s} is unknown. Aborting.".format(
source_disk_id))
if disk_info.getHostId().strip() == getHostName():
msg = "Source disk specified Disk ID: {:s} is inserted in this node: {:s}"
raise Exception(msg.format(source_disk_id, getHostName()))
file_info_list = _get_file_info1(connection, source_disk_id)
source_file_list = []
for file_info in file_info_list:
source_file_list.append(file_info)
del file_info_list
# Get information about files on the target host (.e.g. this host)
# NGAS Utils Functional Tests: Use special target host if requested
file_info_list = _get_file_info2(connection, _get_target_host(),
source_disk_id)
target_file_list = []
for file_info in file_info_list:
target_file_list.append(file_info)
del file_info_list
# Build a dictionary with the target files with (<File ID>, <File Version>) as keys
target_file_dict = {}
for file_info in target_file_list:
target_file_dict[(file_info[INDEX_FILE_ID],
file_info[INDEX_FILE_VERSION])] = file_info
# Now go through each source file and check if it is registered in the DB among the target files
for file_info in source_file_list:
source_file_id = file_info[INDEX_FILE_ID]
source_file_version = file_info[INDEX_FILE_VERSION]
key = (source_file_id, source_file_version)
# Check if target file is present in the DB
if key not in target_file_dict:
check_report += "*** Missing target file in DB: {:s}\n".format(
file_info)
if auto_clone:
check_report += clone_check_file(file_info[INDEX_DISK_ID],
file_info[INDEX_FILE_ID],
file_info[INDEX_FILE_VERSION],
connection, ngams_client)
check_report += "\n"
continue
target_file_info = target_file_dict[key]
mount_point = target_file_info[INDEX_MOUNT_POINT]
filename = target_file_info[INDEX_FILE_NAME]
target_file_path = os.path.normpath(os.path.join(
mount_point, filename))
message = "*** Checking file: {:s}".format(target_file_path)
if auto_clone:
message = "\n" + message
print(message)
# 1. Check that the target file is physically present on the target disk
if not os.path.exists(target_file_path):
check_report += "Missing target file on disk: {:s}\n".format(
str(file_info))
if auto_clone:
check_report += clone_check_file(file_info[INDEX_DISK_ID],
file_info[INDEX_FILE_ID],
file_info[INDEX_FILE_VERSION],
connection, ngams_client)
check_report += "\n"
continue
# 2. Check that the size is correct
source_file_size = file_info[INDEX_FILE_SIZE]
target_file_size = os.path.getsize(target_file_path)
if source_file_size != target_file_size:
check_report += "Wrong size of target file: {:s}\n".format(
str(file_info))
check_report += " - Check file manually!\n"
if check_report:
check_report_main += check_report
else:
check_report_main += "No discrepancies found\n"
return check_report_main
def handleCmd(srvObj,
reqPropsObj,
httpRef):
"""
Handle STATUS command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler object
(ngamsHttpRequestHandler).
Returns: Void.
"""
status = ngamsStatus.ngamsStatus()
status.setDate(toiso8601()).\
setVersion(getNgamsVersion()).setHostId(srvObj.getHostId()).\
setStatus(NGAMS_SUCCESS).\
setMessage("Successfully handled command STATUS").\
setState(srvObj.getState()).setSubState(srvObj.getSubState())
reqPropsObjRef = reqPropsObj
# Get the information requested.
diskId = ""
fileId = ""
fileVersion = ""
configurationFile = ""
fileAccess = ""
hostId = ""
requestId = ""
dbTime = ""
dbTimeReset = ""
fileList = ""
fileListId = ""
maxElements = 100000
if (reqPropsObj.hasHttpPar("disk_id")):
diskId = reqPropsObj.getHttpPar("disk_id")
if (reqPropsObj.hasHttpPar("file_id")):
fileId = reqPropsObj.getHttpPar("file_id")
if (reqPropsObj.hasHttpPar("file_version")):
fileVersion = int(reqPropsObj.getHttpPar("file_version"))
if (reqPropsObj.hasHttpPar("configuration_file")):
configurationFile = reqPropsObj.getHttpPar("configuration_file")
if (reqPropsObj.hasHttpPar("file_access")):
fileAccess = reqPropsObj.getHttpPar("file_access")
if (reqPropsObj.hasHttpPar("host_id")):
hostId = reqPropsObj.getHttpPar("host_id")
if (reqPropsObj.hasHttpPar("max_elements")):
maxElements = int(reqPropsObj.getHttpPar("max_elements"))
if (reqPropsObj.hasHttpPar("request_id")):
requestId = reqPropsObj.getHttpPar("request_id")
if (reqPropsObj.hasHttpPar("db_time")):
dbTime = True
if (reqPropsObj.hasHttpPar("db_time_reset")):
dbTimeReset = True
if (reqPropsObj.hasHttpPar("flush_log")):
# in the past this called flushLog()
# do we really want to keep that functionality? I doubt it
pass
if (reqPropsObj.hasHttpPar("file_list")):
fileList = int(reqPropsObj.getHttpPar("file_list"))
if (reqPropsObj.hasHttpPar("file_list_id")):
fileListId = reqPropsObj.getHttpPar("file_list_id")
if (reqPropsObj.hasHttpPar("dump_object_info")):
# Dump status of all objects allocated into file specified.
targFile = reqPropsObj.getHttpPar("dump_object_info")
rmFile(targFile)
fo = open(targFile, "w")
fo.write(_genObjectStatus())
fo.close()
# Handle request for file info.
genCfgStatus = 0
genDiskStatus = 0
genFileStatus = 0
genStatesStatus = 1
genRequestStatus = 0
msg = ""
help = 0
if (reqPropsObj.hasHttpPar("help")):
global _help
msg = _help
help = 1
elif hostId and hostId != srvObj.getHostId():
host, port = srvObj.get_remote_server_endpoint(hostId)
cfgObj = srvObj.getCfg()
if not cfgObj.getProxyMode():
httpRef.redirect(host, port)
return
else:
try:
httpRef.proxy_request(hostId, host, port)
except Exception as e:
ex = re.sub("<|>", "", str(e))
errMsg = genLog("NGAMS_ER_COM",
[host, port,ex])
raise Exception(errMsg)
return
elif (fileList):
if (not fileListId):
# It's a new STATUS?file_list request.
fileListId = _handleFileList(srvObj, reqPropsObj, httpRef)
# Send back data from the request.
_handleFileListReply(srvObj, reqPropsObj, httpRef, fileListId,
maxElements)
elif (diskId):
diskObj = ngamsDiskInfo.ngamsDiskInfo()
diskObj.read(srvObj.getDb(), diskId)
status.addDiskStatus(diskObj)
genDiskStatus = 1
elif (fileId):
if (not fileVersion): fileVersion = -1
try:
fileObj = ngamsFileInfo.ngamsFileInfo()
fileObj.read(srvObj.getHostId(), srvObj.getDb(), fileId, fileVersion)
diskObj = ngamsDiskInfo.ngamsDiskInfo()
try:
diskObj.read(srvObj.getDb(), fileObj.getDiskId())
except:
errMsg = "Illegal Disk ID found: {0} for file with ID: {1}".format(fileObj.getDiskId(), fileId)
raise Exception(errMsg)
diskObj.addFileObj(fileObj)
status.addDiskStatus(diskObj)
except:
# The file was not found in the database. Check if it is available
# on a remote partner site.
host, port, status_info, disk_info, file_info = \
ngamsFileUtils.lookup_partner_site_file_status(srvObj, fileId,
fileVersion,
reqPropsObj)
# Update status reply using the disk status and file status
# information retrieved from the partner site
status.addDiskStatus(disk_info)
genDiskStatus = 1
genFileStatus = 1
elif (requestId):
logger.debug("Checking status of request with ID: %s", requestId)
reqPropsObjRef = srvObj.getRequest(requestId)
if (not reqPropsObjRef):
errMsg = genLog("NGAMS_ER_ILL_REQ_ID", [requestId])
raise Exception(errMsg)
genRequestStatus = 1
elif (configurationFile):
msg = "configuration_file=" + srvObj.cfg_fname
genCfgStatus = 1
# Hidden feature to return complete Cfg!!
try:
genCfgStatus = int(configurationFile)
except ValueError:
genCfgStatus = 1
status.setNgamsCfgObj(srvObj.getCfg())
elif (fileAccess):
if (not fileVersion): fileVersion = -1
fileId = fileAccess
msg = _checkFileAccess(srvObj, reqPropsObj, httpRef, fileId,
fileVersion, diskId)
elif (dbTime):
logger.debug("Querying total DB time")
msg = "Total DB time: %.6fs" % srvObj.getDb().getDbTime()
elif (dbTimeReset):
msg = "Resetting DB timer"
logger.debug(msg)
srvObj.getDb().resetDbTime()
else:
msg = "Successfully handled command STATUS"
if (reqPropsObjRef == reqPropsObj):
reqPropsObj.setCompletionTime()
srvObj.updateRequestDb(reqPropsObj)
if (genCfgStatus or genDiskStatus or genFileStatus or genRequestStatus):
status.setReqStatFromReqPropsObj(reqPropsObjRef)
# Generate XML reply.
logger.debug("Status requests: " + str([genCfgStatus, genDiskStatus, genFileStatus,
genStatesStatus]))
xmlStat = status.genXmlDoc(genCfgStatus, genDiskStatus, genFileStatus,
genStatesStatus)
xmlStat = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, xmlStat)
httpRef.send_data(six.b(xmlStat), NGAMS_XML_MT)
elif not httpRef.reply_sent:
httpRef.send_status(msg)
if (msg and (not help)):
logger.info(msg)
else:
logger.info("Successfully handled command STATUS")
def checkCorrectFileSizeDb(diskId, correct=0, notifEmail=None):
"""
Check the size of the files registered on the given disk. If a file
has a size different from the one stored in the DB, the size is
corrected in the DB
diskId: ID of disk for which to check the files (string).
Returns: Void
"""
server, db, user, password = ngasUtilsLib.getDbPars()
db = ngamsDb.ngamsDb(server, db, user, password, createSnapshot=0)
tmpDiskInfo = ngamsDiskInfo.ngamsDiskInfo().read(getHostName(), db, diskId)
if (tmpDiskInfo.getHostId() != getHostName()):
raise Exception, "The disk: %s is not inserted in this system!!" %\
diskId
notifMsg = "\nFILE SIZE CHECK/CORRECTION REPORT:\n" +\
"(only files with an incorrect size in the DB are reported)\n"+\
"\nDisk ID: " + diskId + "\n" +\
128 * "="
sys.stdout.write(notifMsg)
dbCursor = db.getFileSummary1(getHostName(), [diskId], fileStatus=[])
while (1):
res = dbCursor.fetch(128)
if (not res): break
for fileInfo in res:
# IMPL: This works only on calibration frames for the moment.
# (these are uncompressed -> file size == uncompr file size)
filename = fileInfo[2]
if (filename.find("/M.") == -1): continue
mountPoint = fileInfo[1]
dbChecksum = fileInfo[3]
fileId = fileInfo[5]
fileVer = fileInfo[6]
dbFileSize = int(fileInfo[7])
diskId = fileInfo[9]
complFilename = os.path.normpath(mountPoint + "/" + filename)
diskFileSize = getFileSize(complFilename)
msg = "\nCheck size of file: %s/%s/%d: " % (diskId, fileId,
fileVer)
if (diskFileSize == dbFileSize):
msg += "Size OK"
else:
msg += "SIZE WRONG!"
# Check checksum.
from ngamsPlugIns import ngamsGenCrc32
checksum = ngamsGenCrc32.ngamsGenCrc32(None, complFilename, 0)
if (checksum != dbChecksum):
msg += " ILLEGAL CHECKSUM!"
else:
msg += " CHECKSUM OK!"
if (correct):
tmpFileInfo = ngamsFileInfo.\
ngamsFileInfo().read(db, fileId, fileVer,
diskId)
tmpFileInfo.setFileSize(diskFileSize)
tmpFileInfo.setUncompressedFileSize(diskFileSize)
tmpFileInfo.setFileStatus(NGAMS_FILE_STATUS_OK)
if (not tmpFileInfo.getCreationDate()):
tmpFileInfo.setCreationDate(tmpFileInfo.\
getIngestionDate())
tmpFileInfo.write(getHostName(), db, 0, 1)
msg += " FILE SIZE CORRECTED!"
notifMsg += msg
sys.stdout.write(msg)
sys.stdout.flush()
notifMsg += "\n%s\n" % (128 * "=")
print "\n" + 128 * "="
if (notifEmail):
ngasUtilsLib.sendEmail("FILE SIZE CHECK/CORRECTION REPORT", notifEmail,
notifMsg, "text/plain",
"FILE-SIZE-CHECK-REP-%s" % diskId)
def _locateArchiveFile(srvObj,
fileId,
fileVersion,
diskId,
hostId,
reqPropsObj,
files,
include_compression):
"""
See description of ngamsFileUtils.locateArchiveFile(). This function is
used simply to encapsulate the complete processing to be able to clean up.
"""
msg = "_locateArchiveFile() - Disk ID: %s - File ID: " +\
"%s - File Version: %d ..."
logger.debug(msg, str(diskId), fileId, int(fileVersion))
# Filter out files not on specified host if host ID is given.
if (hostId):
files = filter(lambda x: x[1] == hostId, files)
# If no file was found we raise an exception.
if not files:
tmpFileRef = fileId
if (fileVersion > 0): tmpFileRef += "/Version: " + str(fileVersion)
if (diskId): tmpFileRef += "/Disk ID: " + diskId
if (hostId): tmpFileRef += "/Host ID: " + hostId
errMsg = genLog("NGAMS_ER_UNAVAIL_FILE", [tmpFileRef])
raise Exception(errMsg)
# We now sort the file information sub-lists in the file list.
# The priori is as follows:
#
# 1. Local host.
# 2. Same cluster.
# 3. Same domain (e.g. hq.eso.org).
# 4. Other files (remote files).
localHostFileList = []
clusterFileList = []
domainFileList = []
remoteFileList = []
all_hosts = set([x[1] for x in files])
hostDic = ngamsHighLevelLib.resolveHostAddress(srvObj.getHostId(),
srvObj.getDb(),
srvObj.getCfg(),
all_hosts)
# Loop over the candidate files and sort them.
fileCount = idx = 0
for fileInfo in files:
fileHost = fileInfo[1]
if (hostDic[fileHost].getHostType() == NGAMS_HOST_LOCAL):
localHostFileList.append(fileInfo)
elif (hostDic[fileHost].getHostType() == NGAMS_HOST_CLUSTER):
clusterFileList.append(fileInfo)
elif (hostDic[fileHost].getHostType() == NGAMS_HOST_DOMAIN):
domainFileList.append(fileInfo)
else:
# NGAMS_HOST_REMOTE:
remoteFileList.append(fileInfo)
idx += 1
fileCount += 1
# The highest priority of the file is determined by the File Version,
# the latest version is preferred even though this may be stored
# on another NGAS host.
# A dictionary is built up, which contains the candidate files. The
# format is such there each version found is one key. For each key
# (= version) there is a list with the corresponding file information
# order according to the location.
candFileDic = {}
fileLists = [[NGAMS_HOST_LOCAL, localHostFileList],
[NGAMS_HOST_CLUSTER, clusterFileList],
[NGAMS_HOST_DOMAIN, domainFileList],
[NGAMS_HOST_REMOTE, remoteFileList]]
for fileListInfo in fileLists:
location = fileListInfo[0]
fileList = fileListInfo[1]
for fileInfo in fileList:
fileVer = fileInfo[0].getFileVersion()
# Create a list in connection with each File Version key.
if fileVer not in candFileDic:
candFileDic[fileVer] = []
candFileDic[fileVer].append([location, fileInfo[0], fileInfo[1]])
fileVerList = list(candFileDic)
fileVerList.sort(reverse=True)
if logger.level <= logging.DEBUG:
msg = ""
count = 1
for fileVer in fileVerList:
for fi in candFileDic[fileVer]:
msg += "(" + str(count) + ": Location:" + fi[0] +\
", Host:" + fi[2] + ", Version:" +\
str(fi[1].getFileVersion()) + ") "
count += 1
logger.debug("File list to check: " + msg)
# If no files were found we raise an exception.
if (len(candFileDic) == 0):
if (fileVersion != -1):
fileRef = fileId + "/V" + str(fileVersion)
else:
fileRef = fileId
errMsg = genLog("NGAMS_ER_UNAVAIL_FILE", [fileRef])
raise Exception(errMsg)
# We generate a list with the Disk IDs (which we need later).
# Generate a dictionary with Disk Info Objects.
diskIdDic = {}
for fileVer in fileVerList:
for fileInfo in candFileDic[fileVer]:
diskIdDic[fileInfo[1].getDiskId()] = fileInfo[1].getDiskId()
sqlDiskInfo = srvObj.getDb().getDiskInfoFromDiskIdList(list(diskIdDic))
diskInfoDic = {}
for diskInfo in sqlDiskInfo:
diskInfoObj = ngamsDiskInfo.ngamsDiskInfo().unpackSqlResult(diskInfo)
diskInfoDic[diskInfoObj.getDiskId()] = diskInfoObj
logger.debug("Disk Info Objects Dictionary: %s", str(diskInfoDic))
# Check if the files are accessible - when the first accessible file
# in the fileList is found, the information is returned as the file wanted.
# To check the file accessibility, it is also checked if the NG/AMS
# 'responsible' for the file, allows for Retrieve Requests (only done
# in connection with a Retrieve Request).
logger.debug("Checking which of the candidate files should be selected ...")
foundFile = 0
for fileVer in fileVerList:
if (foundFile): break
for fileInfo in candFileDic[fileVer]:
location = fileInfo[0]
fileInfoObj = fileInfo[1]
host = fileInfo[2]
diskInfoObj = diskInfoDic[fileInfoObj.getDiskId()]
port = hostDic[host].getSrvPort()
logger.debug("Checking candidate file with ID: %s on host/port: %s/%s. " + \
"Location: %s",
fileInfoObj.getFileId(), host, str(port), location)
# If the file is stored locally we check if it is accessible,
# otherwise we send a STATUS/file_access request to the
# host in question.
if (location == NGAMS_HOST_LOCAL):
# Check first if the local system supports retrieve requests.
# (if relevant).
if (reqPropsObj):
if (reqPropsObj.getCmd() == NGAMS_RETRIEVE_CMD):
if (not srvObj.getCfg().getAllowRetrieveReq()):
continue
# Check if the file is accessible.
filename = os.path.normpath(diskInfoObj.getMountPoint()+"/" +\
fileInfoObj.getFilename())
logger.debug("Checking if local file with name: %s is available", filename)
if (not os.path.exists(filename)):
logger.debug(genLog("NGAMS_INFO_FILE_NOT_AVAIL", [fileId, host]))
else:
logger.debug(genLog("NGAMS_INFO_FILE_AVAIL", [fileId, host]))
foundFile = 1
break
else:
logger.debug("Checking if file with ID/Version: %s/%s " +\
"is available on host/port: %s/%s",
fileInfoObj.getFileId(), str(fileInfoObj.getFileVersion()),
host, str(port))
# If a server hosting a file is suspended, it is woken up
# to be able to check if the file is really accessible.
if (hostDic[host].getSrvSuspended() == 1):
logger.debug("Server hosting requested file (%s/%s) is suspended " + \
"- waking up server ...",
host, str(port))
try:
ngamsSrvUtils.wakeUpHost(srvObj, host)
logger.debug("Suspended server hosting requested file (%s/%s) " +\
"has been woken up",
host, str(port))
except Exception:
logger.exception("Error waking up server hosting selected " +\
"file")
continue
# The file is hosted on a host, which is not suspended or
# which was successfully woken up.
pars = [["file_access", fileInfoObj.getFileId()]]
if (fileInfoObj.getFileVersion() != -1):
pars.append(["file_version", fileInfoObj.getFileVersion()])
ipAddress = hostDic[host].getIpAddress()
authHdr = ngamsSrvUtils.genIntAuthHdr(srvObj)
resp = ngamsHttpUtils.httpGet(ipAddress, port, NGAMS_STATUS_CMD,
pars=pars, auth=authHdr)
with contextlib.closing(resp):
data = resp.read()
statusObj = ngamsStatus.ngamsStatus().unpackXmlDoc(data, 1)
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Result of File Access Query: %s",
re.sub("\n", "", str(statusObj.genXml().toprettyxml(' ', '\n'))))
if ((statusObj.getMessage().\
find("NGAMS_INFO_FILE_AVAIL") == -1)):
logger.debug(genLog("NGAMS_INFO_FILE_NOT_AVAIL", [fileId, host]))
else:
logger.debug(genLog("NGAMS_INFO_FILE_AVAIL", [fileId, host]))
foundFile = 1
break
# If no file was found we raise an exception.
if (not foundFile):
errMsg = genLog("NGAMS_ER_UNAVAIL_FILE", [fileId])
raise Exception(errMsg)
# The file was found, get the info necessary for the acquiring the file.
ipAddress = hostDic[host].getIpAddress()
srcFileInfo = [location, host, ipAddress, port,
diskInfoObj.getMountPoint(),
fileInfoObj.getFilename(), fileInfoObj.getFileId(),
fileInfoObj.getFileVersion(), fileInfoObj.getFormat()]
if include_compression:
srcFileInfo.append(fileInfoObj.getCompression())
msg = "Located suitable file for request - File ID: %s. " +\
"Info for file found - Location: %s - Host ID/IP: %s/%s - " +\
"Port Number: %s - File Version: %d - Filename: %s - " +\
"Mime-type: %s"
logger.debug(msg, fileId, location, host, ipAddress, port,
fileInfoObj.getFileVersion(), fileInfoObj.getFilename(),
fileInfoObj.getFormat())
return srcFileInfo
httpRef.proxy_request(hostId, host, port)
except Exception, e:
ex = re.sub("<|>", "", str(e))
errMsg = genLog("NGAMS_ER_COM",
[host, port,ex])
raise Exception, errMsg
return
elif (fileList):
if (not fileListId):
# It's a new STATUS?file_list request.
fileListId = _handleFileList(srvObj, reqPropsObj, httpRef)
# Send back data from the request.
_handleFileListReply(srvObj, reqPropsObj, httpRef, fileListId,
maxElements)
elif (diskId):
diskObj = ngamsDiskInfo.ngamsDiskInfo()
diskObj.read(srvObj.getDb(), diskId)
status.addDiskStatus(diskObj)
genDiskStatus = 1
elif (fileId):
if (not fileVersion): fileVersion = -1
fileObj = ngamsFileInfo.ngamsFileInfo()
fileObj.read(srvObj.getHostId(), srvObj.getDb(), fileId, fileVersion)
diskObj = ngamsDiskInfo.ngamsDiskInfo()
try:
diskObj.read(srvObj.getDb(), fileObj.getDiskId())
except:
errMsg = "Illegal Disk ID found: %s for file with ID: %s" %\
(fileObj.getDiskId(), fileId)
raise Exception, errMsg
diskObj.addFileObj(fileObj)
def register(srvObj, path, mimeType="", reqPropsObj=None, httpRef=None):
"""
Function to generate a list of candidate files to register, and to
launch a thread that actually carries out the registration.
When the possible candidate files haven been selected, a reply is sent
back to the requestor if the 'httpRef' object is given.
srvObj: Instance of NG/AMS Server object (ngamsServer).
path: The path name, which is used as starting point for
the searching for files (string).
mimeType: Comma separated list of mime-types, which should be
considered for registration (string).
reqPropsObj: If an NG/AMS Request Properties Object is given, the
Request Status will be updated as the request is carried
out (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
# Check if the given path or file exists.
if (not os.path.exists(path)):
errMsg = genLog("NGAMS_ER_FILE_REG_FAILED",
[path, "Non-existing file or path."])
raise Exception(errMsg)
# Check if the given path/file is on one of the NGAS Disks.
foundPath = 0
for slotId in srvObj.getDiskDic().keys():
do = srvObj.getDiskDic()[slotId]
if (path.find(do.getMountPoint()) == 0):
foundPath = 1
break
if (not foundPath):
errMsg = genLog("NGAMS_ER_FILE_REG_FAILED", [
path, "File or path specified is not located on an "
"NGAS Disk."
])
raise Exception(errMsg)
# Create file pattern for temporary files.
tmpFilePat = ngamsHighLevelLib.genTmpFilename(srvObj.getCfg(),
"REGISTER_CMD")
# Generate dictionary with mime-types to accept.
mimeTypeDic = {}
if (mimeType != ""):
for mt in mimeType.split(","):
try:
mimeTypeDic[mt] = srvObj.getMimeTypeDic()[mt]
except:
errMsg = genLog("NGAMS_ER_REQ_HANDLING",
["Mime-type specified: " + mt + " " +\
"in connection with REGISTER command " +\
"does not have a DAPI defined"])
raise Exception(errMsg)
else:
mimeTypeDic = srvObj.getMimeTypeDic()
# From the Disk Dictionary, generate a dictionary with mappings from
# Disk ID + from Mount Point to the disk information for the disk.
diskInfoDic = {}
mtPt2DiskInfo = {}
for slotId in srvObj.getDiskDic().keys():
if (not srvObj.getCfg().getSlotIdDefined(slotId)): continue
diskId = srvObj.getDiskDic()[slotId].getDiskId()
mtPt = srvObj.getDiskDic()[slotId].getMountPoint()
tmpDiskInfo = ngamsDiskInfo.ngamsDiskInfo().\
read(srvObj.getDb(), diskId)
diskInfoDic[diskId] = tmpDiskInfo
mtPt2DiskInfo[mtPt] = tmpDiskInfo
# Generate a list with all files found under the specified path, which
# are candidates for being registered.
fileListDbmName = tmpFilePat + "_FILE_LIST"
rmFile(fileListDbmName + "*")
fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm=1)
mimeTypeMappings = srvObj.getCfg().getMimeTypeMappings()
tmpGlobFile = tmpFilePat + "_FILE_GLOB.glob"
fileCount = 0
searchPath = os.path.normpath(path)
foundVolume = False
for mtPt in mtPt2DiskInfo.keys():
mtPt2 = mtPt
if (mtPt[-1] != "/"): mtPt2 += "/"
if (searchPath.find(mtPt2) == 0):
foundVolume = True
if os.path.isdir(searchPath):
for root, dirs, files in os.walk(searchPath):
for nextFile in files:
if nextFile not in \
[NGAMS_DISK_INFO, \
NGAMS_VOLUME_ID_FILE, \
NGAMS_VOLUME_INFO_FILE]:
mimeType = ngamsLib.detMimeType(
mimeTypeMappings, nextFile, 0)
tmpFileInfo = [
os.path.join(root, nextFile),
mtPt2DiskInfo[mtPt].getDiskId(), mimeType
]
fileListDbm.add(os.path.join(root, nextFile),
tmpFileInfo)
elif os.path.isfile(
searchPath): # the path is actually pointing to a file
nextFile = os.path.basename(searchPath)
root = os.path.dirname(searchPath)
if nextFile not in \
[NGAMS_DISK_INFO, \
NGAMS_VOLUME_ID_FILE, \
NGAMS_VOLUME_INFO_FILE]:
mimeType = ngamsLib.detMimeType(mimeTypeMappings, nextFile,
0)
tmpFileInfo = [
os.path.join(root, nextFile),
mtPt2DiskInfo[mtPt].getDiskId(), mimeType
]
fileListDbm.add(os.path.join(root, nextFile), tmpFileInfo)
if foundVolume: break
# pattern = ""
# # TODO: Portatibility issue. The usage of UNIX commands should be
# # avoided if possible.
# while (1):
# # Use a shell commands here to avoid building up maybe huge lists in
# # memory/in the Python interpreter. A better way could maybe be found
# # avoiding to invoke a shell command.
# rmFile(tmpGlobFile)
# searchPath = os.path.normpath(path + pattern)
# tmpCmd = "find " + searchPath + " -maxdepth 1 > " + tmpGlobFile
# stat, out = commands.getstatusoutput(tmpCmd)
# if (stat != 0): break
# fo = open(tmpGlobFile)
# while (1):
# nextFile = fo.readline().strip()
# if (nextFile == ""): break
# if (nextFile.find("/" + NGAMS_DISK_INFO) != -1): continue
# if (nextFile.find("/" + NGAMS_VOLUME_ID_FILE) != -1): continue
# if (nextFile.find("/" + NGAMS_VOLUME_INFO_FILE) != -1): continue
# if (os.path.isfile(nextFile)):
# nextFile = os.path.normpath(nextFile)
#
# # Find the host disk. Only files located on mounted NGAS
# # disks are considered.
# for mtPt in mtPt2DiskInfo.keys():
# mtPt2 = mtPt
# if (mtPt[-1] != "/"): mtPt2 += "/"
# if (nextFile.find(mtPt2) == 0):
# info(4,"Found candidate file for registering: " +\
# nextFile)
# # Take only files with a Registration Plug-In defined.
# mimeType = ngamsLib.detMimeType(mimeTypeMappings,
# nextFile, 0)
# regPi = srvObj.getCfg().getRegPiFromMimeType(mimeType)
# if (regPi != None):
# tmpFileInfo = [nextFile,
# mtPt2DiskInfo[mtPt].getDiskId(),
# mimeType]
# fileListDbm.add(nextFile, tmpFileInfo)
# break
# fo.close()
# pattern += "/*"
# rmFile(tmpGlobFile)
fileListDbm.sync()
del fileListDbm
# Send intermediate reply if the HTTP Reference object is given.
is_async = 'async' in reqPropsObj and int(reqPropsObj['async'])
if httpRef and is_async:
logger.debug("REGISTER command accepted - generating immediate " +\
"confimation reply to REGISTER command")
# Update the request status in the Request Properties Object.
reqPropsObj.\
setExpectedCount(fileCount).\
setActualCount(0).setCompletionPercent(0)
srvObj.updateRequestDb(reqPropsObj)
status = srvObj.genStatus(NGAMS_SUCCESS,
"Accepted REGISTER command for execution").\
setReqStatFromReqPropsObj(reqPropsObj).\
setActualCount(0)
# Launch the register thread or run the command in foreground if wait=1
if is_async:
args = (srvObj, fileListDbmName, tmpFilePat, diskInfoDic, reqPropsObj,
None)
thrName = NGAMS_REGISTER_THR + threading.current_thread().getName()
regThread = threading.Thread(None, _registerThread, thrName, args)
regThread.setDaemon(0)
regThread.start()
else:
# Carry out the REGISTER Command (directly in this thread) and send
# reply when this is done.
_registerExec(srvObj, fileListDbmName, tmpFilePat, diskInfoDic,
reqPropsObj)
msg = "Successfully handled command REGISTER"
logger.debug(msg)
status = srvObj.genStatus(NGAMS_SUCCESS, msg).\
setReqStatFromReqPropsObj(reqPropsObj).setActualCount(0)
# Send reply if possible.
if (httpRef):
xmlStat = status.genXmlDoc(0, 0, 0, 1, 0)
xmlStat = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, xmlStat)
httpRef.send_data(six.b(xmlStat), NGAMS_XML_MT)
def _remDisk(srvObj, reqPropsObj, diskId, execute, tmpFilePat):
"""
See documentation for the ngamsRemDiskCmd.remDisk() function.
"""
# Get disk info -- if any available.
sqlDiskInfo = srvObj.getDb().getDiskInfoFromDiskId(diskId)
diskInfo = ngamsDiskInfo.ngamsDiskInfo()
if (sqlDiskInfo != []):
if (diskInfo.unpackSqlResult(sqlDiskInfo).getHostId() !=
srvObj.getHostId()):
sqlDiskInfo = None
# Check that the disk is mounted in this unit (no proxy for the REMDISK
# Command ...).
if (diskInfo.getHostId() != srvObj.getHostId()):
errMsg = "Disk referred to by Disk ID: %s seems not to be mounted " +\
"in this unit: %s -- rejecting REMDISK Command"
errMsg = errMsg % (diskId, srvObj.getHostId())
raise Exception(errMsg)
# Check that execution of the request can be granted.
filesMisCopyDbmName, filesNotRegDbmName, fileListDbmName =\
ngamsRemUtils.checkFileCopiesAndReg(srvObj, 3,
tmpFilePat, None,
diskId)
status = ngamsRemUtils._remStatErrReport(srvObj, reqPropsObj, tmpFilePat,
filesMisCopyDbmName,
filesNotRegDbmName,
fileListDbmName, diskId)
misCopiesDbm = ngamsDbm.ngamsDbm(filesMisCopyDbmName)
if (misCopiesDbm.getCount() == 0):
logger.info("Disk with ID: %s approved for deletion", diskId)
else:
logger.info("Disk with ID: %s rejected for deletion", diskId)
if (status): return status
# Check if there is enough space on the disk to store the file info
# cache files.
kbPerFile = 0.256
tmpDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm=0)
noOfFiles = tmpDbm.getCount()
del tmpDbm
kbsAvail = getDiskSpaceAvail(diskInfo.getMountPoint(), format="KB")
kbsReq = (kbPerFile * noOfFiles)
msg = "Space required for REMDISK Command: %.1f KB, " +\
"space available: %.1f KB"
logger.debug(msg, kbsReq, kbsAvail)
if (kbsReq > kbsAvail):
errMsg = "Not enough space on disk to carry out REMDISK Command. " +\
"Host: %s, Disk ID: %s, Mount Point: %s. " +\
"Required disk space: %.3f MB"
errMsg = errMsg % (getHostName(), diskInfo.getDiskId(),
diskInfo.getMountPoint(), (kbsReq / 1024.))
raise Exception(errMsg)
#########################################################################
# Execute the deletion if execute = 1 and a matching disk was found.
# Otherwise just return the ngamsStatus object as confirmation.
#########################################################################
if (execute and (sqlDiskInfo != [])):
# We delete first the info in the DB in connection with the disk and
# afterwards the contents on the disk. This is done like this, since
# it is considered worse having entries for files in the DB which are
# not available on the disk since NGAS uses the info in the DB to check
# for the number of available copies.
logger.info("Removing DB record for disk with ID: %s", diskId)
try:
tmpDir = os.path.dirname(tmpFilePat)
srvObj.getDb().deleteDiskInfo(diskId, 1)
except Exception as e:
errMsg = genLog("NGAMS_ER_DEL_DISK_DB", [diskId, str(e)])
raise Exception(errMsg)
logger.info("Deleting contents on disk with ID: %s", diskId)
try:
rmFile(os.path.normpath(diskInfo.getMountPoint() + "/*"))
except Exception as e:
errMsg = genLog("NGAMS_ER_DEL_DISK", [diskId, str(e)])
raise Exception(errMsg)
try:
# Remember to remove entry for disk from the Physical Disk Dic.
del srvObj.getDiskDic()[diskInfo.getSlotId()]
except:
pass
infoMsg = genLog("NGAMS_INFO_DEL_DISK", [diskId])
logger.info(infoMsg)
# Add entry in the NGAS Disks History Table.
ngasDiskInfo = ngamsDiskUtils.prepNgasDiskInfoFile(
srvObj.getHostId(), diskInfo, 1, 1)
srvObj.getDb().addDiskHistEntry(srvObj.getHostId(), diskId,
"Disk Removed", NGAMS_XML_MT,
ngasDiskInfo)
elif (sqlDiskInfo != []):
infoMsg = genLog("NGAMS_INFO_DEL_DISK_SEL", [diskId])
else:
infoMsg = genLog("NGAMS_WA_DEL_DISK2", [srvObj.getHostId(), diskId])
# Generate status.
status = srvObj.genStatus(NGAMS_SUCCESS, infoMsg)
if (sqlDiskInfo): status.addDiskStatus(diskInfo)
#########################################################################
return status
def checkFileCopiesAndReg(srvObj,
minReqCopies,
dbFilePat,
fileListDbmName=None,
diskId=None,
ignoreMounted=0):
"""
The function checks for each file referenced if there are at least
'minReqCopies' copies available somewhere in this NGAS cluster. For the
files where this is not the case, an entry is added in a ngasDiskInfo
object indicating that this file
If an entire disk is analysed (about to be deleted), it is also checked if
each file stored on the disk is registered in the DB. Otherwise an NG/AMS
File List is returned, containing references to the files not registered.
srvObj: Instance of the NG/AMS Server Class (ngamsServer).
minReqCopies: Minimum number of copies required (integer).
dbFilePat: Filename pattern used to build the DBM containing
information about the files (string).
fileListDbmName: Name of DBM DB containing explicit references to files
to be checked if they can be deleted. The information
in this table is pickled lists with the lay-out defined
by ngamsDb._ngasFilesCols (string).
diskId: Used to refer to all files stored on a disk (string|None).
ignoreMounted: Carry out the check also if the disk is not mounted
(integer/0|1).
Returns: Tuple contaning the filenames of three DBM DBs with the
following information:
o Files not having the specified number of copies.
The contents of this DB are keys (Disk ID + File ID +
File Version), pointing to pickled ngamsFileInfo
objects.
o Files found on the disk but not registered.
This contains the complete filenames of files found
on the disk, which are not registered. These filenames
are the keys of this DBM DB.
o Complete list of files referenced in connection
with the query. The contents of this DB are keys,
which are a simple counter pointing to pickled
list containing the information as returned by
ngamsDb.getFileSummary1().
(tuple/string).
"""
T = TRACE()
if ((not fileListDbmName) and (not diskId)):
errMsg = "ngamsSrvUtils.checkFileCopiesAndReg(): Must specify " +\
"either a DBM with files to be checked or a Disk ID"
logger.warning(errMsg)
raise Exception(errMsg)
# Create DBMs:
# - DB containing information about files having less then the
# specified number of copies.
fileMisCopyDbmName = os.path.normpath(dbFilePat + "_MISSING_COPIES")
fileMisCopyDbm = ngamsDbm.ngamsDbm(fileMisCopyDbmName, writePerm=1)
# - DB that contains information about files stored on the DB,
# which are not registered in the NGAS DB. At the end of the function,
# this will contain information about files found on the disk but
# not registered in the NGAS DB.
filesOnDiskDicDbmName = os.path.normpath(dbFilePat + "_FILES_ON_DISK")
filesOnDiskDicDbm = ngamsDbm.ngamsDbm(filesOnDiskDicDbmName, writePerm=1)
# - DB with information about files referenced by the query.
if (not fileListDbmName):
locFileListDbmName = os.path.normpath(dbFilePat + "_FILE_LIST")
fileListDbm = ngamsDbm.ngamsDbm(locFileListDbmName, writePerm=1)
else:
fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm=0)
# - Temporary DBM containing information about all File IDs defined
# by the query.
fileIdDbmName = os.path.normpath(dbFilePat + "_FILE_IDS")
fileIdDbm = ngamsDbm.ngamsDbm(fileIdDbmName, writePerm=1)
# - Temporary DBM containing information about all files available in
# the system with the File ID/File Version defined by the query.
complFileListDbmName = os.path.normpath(dbFilePat + "_COMPL_FILE_LIST")
complFileListDbm = ngamsDbm.ngamsDbm(complFileListDbmName, writePerm=1)
# - Temporary DBM that is used to figure out the number of independent
# copies of each file concerned by the query.
checkDicDbmName = os.path.normpath(dbFilePat + "_CHECK_DIC")
checkDicDbm = ngamsDbm.ngamsDbm(checkDicDbmName, writePerm=1)
# A Disk ID but no file references are given. Retrieve information
# about files concerned from the DB.
if (diskId):
logger.debug("Retrieving information about files on disk with ID: %s",
diskId)
for f in srvObj.db.getFileSummary1(None, [diskId], [],
ignore=0,
fileStatus=[]):
fileListDbm.addIncKey(f)
fileId = f[ngamsDbCore.SUM1_FILE_ID]
fileIdDbm.add(str(fileId), "")
fileVersion = f[ngamsDbCore.SUM1_VERSION]
fileKey = ngamsLib.genFileKey(None, fileId, fileVersion)
checkDicDbm.add(fileKey, {})
fileListDbm.sync()
fileIdDbm.sync()
checkDicDbm.sync()
# Get the list of files located on the disk. Later on, remove entries
# from this dictionary as the files are parsed, based on their DB info,
# further down in this method.
#
# Key in this dictionary is the complete filename of the file.
logger.debug("Get list of files stored on disk ...")
tmpDiskInfo = srvObj.getDb().getDiskInfoFromDiskId(diskId)
diskInfoObj = ngamsDiskInfo.ngamsDiskInfo().\
unpackSqlResult(tmpDiskInfo)
if ((not ignoreMounted) and (not diskInfoObj.getMounted())):
errMsg = "Rejecting request for removing disk with ID: " +\
diskId + " - disk not mounted!"
raise Exception(errMsg)
if (not ignoreMounted):
basePath = os.path.normpath(diskInfoObj.getMountPoint())
pattern = "/*"
logger.debug(
"Generating list with files on disk with base path: %s",
basePath)
while (1):
tmpFileList = glob.glob(basePath + pattern)
if (len(tmpFileList) == 0):
break
else:
for filename in tmpFileList:
if (os.path.isfile(filename) and
(os.path.basename(filename) != NGAMS_DISK_INFO)
and (os.path.basename(filename) !=
NGAMS_VOLUME_ID_FILE)
and (os.path.basename(filename) !=
NGAMS_VOLUME_INFO_FILE)):
filesOnDiskDicDbm.add(str(filename), "")
pattern += "/*"
# Generate File ID DBM in case a file list DBM is given.
if (fileListDbmName):
logger.debug("Handling file list DBM given in the function call ...")
fileListDbm.initKeyPtr()
while (1):
key, tmpFileInfo = fileListDbm.getNext()
if (not key): break
# Update the File ID DBM.
fileId = tmpFileInfo[ngamsDbCore.SUM1_FILE_ID]
fileIdDbm.add(str(fileId), "")
# Update the DBM with references to File ID/Version sets.
fileVersion = tmpFileInfo[ngamsDbCore.SUM1_VERSION]
fileKey = ngamsLib.genFileKey(None, fileId, fileVersion)
checkDicDbm.add(fileKey, {})
fileIdDbm.sync()
checkDicDbm.sync()
# We need to generate a list with all files available in the system
# with the given File ID/File Version.
logger.debug("Retrieving information about all files available with the " +\
"File ID/File Version as defined by the query")
# Due to the limitation of the size of SQL queries, we have to split up
# the SQL query in several sub-queries. The max. length of an SQL query
# is defined by NGAMS_MAX_SQL_QUERY_SZ, we subtract 512 from this for
# the general part of the query, and for each filename we calculate a
# length of len(File ID) + 4 as contribution to the SQL query.
maxQuerySize = (NGAMS_MAX_SQL_QUERY_SZ - 512)
queryIds = []
querySize = 0
noOfFileIds = fileIdDbm.getCount()
fileIdCount = 0
fileIdDbm.initKeyPtr()
fileId = "INIT"
while (fileId):
fileId, dummy = fileIdDbm.getNext()
if (fileId):
queryIds.append(utils.b2s(fileId))
fileIdCount += 1
querySize += (len(fileId) + 4)
if queryIds and (querySize >= maxQuerySize
or fileIdCount == noOfFileIds):
for f in srvObj.db.getFileSummary1(fileIds=queryIds,
fileStatus=[]):
# Take only a sub-result if that File ID + Version
# is concerned by the query.
tmpFileId = f[ngamsDbCore.SUM1_FILE_ID]
tmpFileVersion = f[ngamsDbCore.SUM1_VERSION]
tmpFileKey = ngamsLib.genFileKey(None, tmpFileId,
tmpFileVersion)
if (checkDicDbm.hasKey(tmpFileKey)):
complFileListDbm.addIncKey(f)
complFileListDbm.sync()
queryIds = []
querySize = 0
# Now, go through the files found and order these such that we end up with
# a Dictionary with "<File ID>_<File Version>" as keys referring
# to a dictionary with the Disk IDs of the disks hosting the files as
# keys, and the information for each file on that disk as a tupple.
#
# It is the intention to figure out how many copies we have of each file
# identified by File ID + File Version stored ON DIFFERENT STORAGE MEDIAS
# + on different hosts.
logger.debug("Generate DBM DB with info about independent file copies ...")
complFileListDbm.initKeyPtr()
while (1):
fileKey, fileInfo = complFileListDbm.getNext()
if (not fileKey): break
checkDicKey = ngamsLib.genFileKey(None,
fileInfo[ngamsDbCore.SUM1_FILE_ID],
fileInfo[ngamsDbCore.SUM1_VERSION])
tmpDic = checkDicDbm.get(checkDicKey)
tmpDic[fileInfo[ngamsDbCore.SUM1_DISK_ID]] = fileInfo
checkDicDbm.add(checkDicKey, tmpDic)
# Check if there are at least minReqCopies occurrences of the files +
# check that all files are registered (if a Disk ID is specified).
logger.debug("Check for files with less copies than: %s",
str(minReqCopies))
checkDicDbm.initKeyPtr()
while (1):
checkDicKey, tmpDic = checkDicDbm.getNext()
if (not checkDicKey): break
tmpDicKeys = list(tmpDic)
noOfCopies = len(tmpDicKeys)
if (noOfCopies < minReqCopies):
tmpFileInfo = tmpDic[tmpDicKeys[0]]
fileId = tmpFileInfo[ngamsDbCore.SUM1_FILE_ID]
fileVersion = tmpFileInfo[ngamsDbCore.SUM1_VERSION]
tmpFileObj = ngamsFileInfo.ngamsFileInfo().\
setFileId(fileId).\
setFileVersion(fileVersion).\
setTag("Independent copies: " + str(noOfCopies))
fileKey = ngamsLib.genFileKey(None, fileId, fileVersion)
fileMisCopyDbm.add(str(fileKey), tmpFileObj)
# Remove this file from the Files On Disk DBM - do this only
# if a Disk ID is specified.
if (diskId):
if (diskId in tmpDic):
fileInfo = tmpDic[diskId]
filename = os.path.\
normpath(fileInfo[ngamsDbCore.SUM1_MT_PT] +\
"/" + fileInfo[ngamsDbCore.SUM1_FILENAME])
filename = str(filename)
if (filesOnDiskDicDbm.hasKey(filename)):
filesOnDiskDicDbm.rem(filename)
# Close all DBM objects.
del fileMisCopyDbm
del filesOnDiskDicDbm
del fileListDbm
del fileIdDbm
del complFileListDbm
del checkDicDbm
# The DBM filesOnDiskDicDbmName now contains references to files,
# which are found on the disk but not registered in the DB.
return (fileMisCopyDbmName, filesOnDiskDicDbmName, complFileListDbmName)
def ingestExtFiles(fileListFile, notifEmail):
"""
Ingest the list of files given in the file referred to.
The files are listed as:
<Disk ID> <File ID> [<File Version>]
...
fileListFile: Filename of file containing the file list (string).
notifEmail: List of email addresses to inform about the
execution of the discation procedure (string).
Returns: Void.
"""
fileInfoList = parseFileList(fileListFile)
# Open DB connection.
server, db, user, password = ngasUtilsLib.getDbPars()
dbCon = ngamsDb.ngamsDb(server, db, user, password, 0)
# Find distinct Disk IDs.
diskIdDic = {}
for fileInfo in fileInfoList:
diskIdDic[fileInfo[0]] = None
info(1, "Disk IDs: " + str(diskIdDic.keys()))
# Register disks referred to as external disks in the DB.
dbDiskDic = {}
for diskId in diskIdDic.keys():
if (not dbDiskDic.has_key(diskId)):
diskInfo = dbCon.getDiskInfoFromDiskId(diskId)
if (diskInfo != []):
diskInfoObj = ngamsDiskInfo.ngamsDiskInfo().\
unpackSqlResult(diskInfo)
else:
# Create a new entry for that disk in the DB.
info(1,"Creating new entry for external disk with ID: " +\
str(diskId))
diskInfoObj = ngamsDiskInfo.ngamsDiskInfo().\
setArchive("EXTERNAL").\
setDiskId(diskId).\
setLogicalName(diskId).\
setHostId("").\
setSlotId("").\
setMounted(0).\
setMountPoint("EXTERNAL").\
setNumberOfFiles(0).\
setAvailableMb(0).\
setBytesStored(0).\
setCompleted(1).\
setCompletionDateFromSecs(0).\
setType("EXTERNAL").\
setManufacturer("UNKNOWN").\
setInstallationDateFromSecs(time.time()).\
setChecksum(0).\
setTotalDiskWriteTime(0).\
setLastCheckFromSecs(0).\
setLastHostId("").\
setStorageSetId(diskId)
diskInfoObj.write(dbCon)
diskIdDic[diskId] = diskInfoObj
# Loop over the files and register them in the DB.
fileRegList = []
sys.stdout.write("Registering files ...")
fileCount = 0
for fileInfo in fileInfoList:
diskId = fileInfo[0]
fileId = fileInfo[1]
fileVer = fileInfo[2]
fileInfoObj = ngamsFileInfo.ngamsFileInfo().\
setDiskId(diskId).\
setFilename(fileId).\
setFileId(fileId).\
setFileVersion(fileVer).\
setFormat("").\
setFileSize(0).\
setUncompressedFileSize(0).\
setCompression("").\
setIngestionDateFromSecs(0).\
setIgnore(0).\
setChecksum("").\
setChecksumPlugIn("").\
setFileStatus(NGAMS_FILE_STATUS_OK).\
setCreationDateFromSecs(0).\
setTag("EXTERNAL")
fileInfoObj.write(getHostName(), dbCon, 0, 1)
fileRegList.append((diskId, fileId, fileVer))
time.sleep(0.050)
fileCount += 1
if ((fileCount % 10) == 0): sys.stdout.write(".")
sys.stdout.write("\n")
info(1, "Registered %d files" % fileCount)
report = genReport(fileRegList)
if (notifEmail):
ngasUtilsLib.sendEmail("ngasRegisterExtFiles: FILE REGISTRATION " +\
"REPORT", notifEmail, report)
