def getDiskInfoObjsFromMimeType(hostId,
dbConObj,
ngamsCfgObj,
mimeType,
sendNotification=1):
"""
Get the Disk Info for the disks allocated related to a certain
mime-type/stream.
dbConObj: DB connection object (ngamsDb).
ngamsCfgObj: NG/AMS Configuration Object (ngamsConfig).
mimeType: Mime-type (string).
sendNotification: 1 = send email notification message in case
no disks are found (integer).
Returns: List of Disk Info Objects (list/ngamsDiskInfo).
"""
T = TRACE()
stream = ngamsCfgObj.getStreamFromMimeType(mimeType)
if (stream == None):
errMsg = genLog("NGAMS_AL_NO_STO_SETS", [mimeType])
raise Exception, errMsg
slotIds = []
for id in stream.getStorageSetIdList():
set = ngamsCfgObj.getStorageSetFromId(id)
slotIds.append(set.getMainDiskSlotId())
if (set.getRepDiskSlotId() != ""):
slotIds.append(set.getRepDiskSlotId())
diskInfo = dbConObj.getDiskInfoForSlotsAndHost(hostId, slotIds)
if (diskInfo == []):
errMsg = genLog("NGAMS_AL_NO_STO_SETS", [mimeType])
logger.warning(errMsg)
if (sendNotification):
ngamsNotification.notify(hostId, ngamsCfgObj, NGAMS_NOTIF_NO_DISKS,
"NO STORAGE SET (DISKS) AVAILABLE",
errMsg)
raise Exception, errMsg
# Unpack the disk information into ngamsDiskInfo objects.
diskInfoObjs = []
for diRaw in diskInfo:
tmpDiskInfo = ngamsDiskInfo.ngamsDiskInfo().unpackSqlResult(diRaw)
diskInfoObjs.append(tmpDiskInfo)
return diskInfoObjs
def parseRawPlugInPars(rawPars):
"""
Parse the plug-in parameters given in the NG/AMS Configuration
and return a dictionary with the values.
rawPars: Parameters given in connection with the plug-in in
the configuration file (string).
Returns: Dictionary containing the parameters from the plug-in
parameters as keys referring to the corresponding
value of these (dictionary).
"""
# Plug-In Parameters. Expect:
# "<field name>=<field value>[,field name>=<field value>]"
parDic = {}
pars = string.split(rawPars, ",")
for par in pars:
if (par != ""):
try:
parVal = string.split(par, "=")
par = parVal[0].strip()
parDic[par] = parVal[1].strip()
logger.debug("Found plug-in parameter: %s with value: %s",
par, parDic[par])
except:
errMsg = genLog("NGAMS_ER_PLUGIN_PAR", [rawPars])
raise Exception, errMsg
logger.debug("Generated parameter dictionary: %s", str(parDic))
return parDic
def read(self, hostId, dbConObj, fileId, fileVersion=1, diskId=None):
"""
Query information about a specific file and set the class member
variables accordingly.
dbConObj: DB connection object (ngamsDb).
fileId: File ID (string).
fileVersion: Version of the file to query info for (integer).
diskId: Used to refer to a specific disk (string).
Returns: Reference to object itself.
"""
fileInfo = dbConObj.getFileInfoFromFileIdHostId(
hostId, fileId, fileVersion, diskId)
if (not fileInfo):
fileInfo = dbConObj.getFileInfoFromFileId(fileId,
fileVersion,
diskId,
dbCursor=False)
if fileInfo:
fileInfo = fileInfo[0]
if (fileInfo == []):
errMsg = genLog("NGAMS_ER_UNAVAIL_FILE", [fileId])
raise Exception, errMsg
else:
self.unpackSqlResult(fileInfo)
return self
def _addHostInDic(dbConObj, hostId, hostDic):
"""
Internal function to add host information in a dictionary.
dbConObj: DB object used when accessing the DB (ngamsDb).
hostId: ID of host to add (string).
hostDic: Dictionary with host IDs as keys pointing to instances
of ngamsHostInfo (dictionary).
Returns: Void.
"""
tmpHostInfo = dbConObj.getHostInfoFromHostIds([hostId])
if not tmpHostInfo:
raise Exception, genLog("NGAMS_AL_MIS_HOST", [hostId])
sqlHostInfo = tmpHostInfo[0]
hostDic[hostId] = ngamsHostInfo.ngamsHostInfo().\
unpackFromSqlQuery(sqlHostInfo)
def copyFile(ngamsCfgObj, srcFileSlotId, trgFileSlotId, srcFilename,
trgFilename):
"""
Copy a file, possibly between two different disks.
ngamsCfgObj: Instance of NG/AMS Configuration (ngamsConfig).
srcFileSlotId: Slot ID for source file (string).
trgFileSlotId: Slot ID for target file (string).
srcFilename: Source file filename (string).
trgFilename: Target file filename (string).
Returns: Tuple. Element 0: Time in took to move
file (s) (tuple).
"""
logger.debug("Copying file: %s to filename: %s ...", srcFilename,
trgFilename)
try:
# Make mutual exclusion on disk access (if requested).
acquireDiskResource(ngamsCfgObj, srcFileSlotId)
if (srcFileSlotId != trgFileSlotId):
acquireDiskResource(ngamsCfgObj, trgFileSlotId)
try:
checkCreatePath(os.path.dirname(trgFilename))
fileSize = getFileSize(srcFilename)
checkAvailDiskSpace(trgFilename, fileSize)
start = time.time()
# Make target file writable if existing.
if (os.path.exists(trgFilename)):
os.chmod(trgFilename, 420)
shutil.copyfile(srcFilename, trgFilename)
deltaTime = time.time() - start
logger.debug("File: %s copied to filename: %s", srcFilename,
trgFilename)
except Exception, e:
errMsg = genLog(
"NGAMS_AL_CP_FILE",
[srcFilename, trgFilename, str(e)])
raise Exception, errMsg
# Release disk resouces
releaseDiskResource(ngamsCfgObj, srcFileSlotId)
if (srcFileSlotId != trgFileSlotId):
releaseDiskResource(ngamsCfgObj, trgFileSlotId)
return [deltaTime]
def load(self, xmlDoc):
"""
Load an NG/AMS Configuration from an XML document.
xmlDoc: Name of XML document (string).
Returns: Reference to object itself.
"""
T = TRACE()
try:
self.__xmlMgr.load(xmlDoc)
self.setXmlDoc(xmlDoc)
except Exception, e:
errMsg = genLog("NGAMS_ER_LOAD_CFG", [xmlDoc, str(e)])
raise Exception, errMsg
def load(self, xmlDoc):
"""
Load an NG/AMS Configuration from an XML document.
xmlDoc: Name of XML document (string).
Returns: Reference to object itself.
"""
T = TRACE()
try:
fd = open(self.setXmlDoc(xmlDoc).getXmlDoc())
doc = fd.read()
fd.close()
except Exception, e:
errMsg = genLog("NGAMS_ER_LOAD_CFG", [xmlDoc, str(e)])
raise Exception, errMsg
def getStorageSetIdFromDiskId(dbConObj, ngamsCfgObj, diskId):
"""
Return a Storage Set ID from a Disk ID.
dbConObj: DB connection object (ngamsDb).
ngamsCfgObj: NG/AMS Configuration object (ngamsConfig).
diskId: Disk ID (string).
Returns: Storage Set ID for the disk referred to (string).
"""
# Disk ID -> Slot ID -> Storage Set ID.
slotId = dbConObj.getSlotIdFromDiskId(diskId)
if (slotId == None):
errMsg = genLog("NGAMS_ER_MISSING_DISK_ID", [diskId])
raise Exception(errMsg)
set = ngamsCfgObj.getStorageSetFromSlotId(slotId)
return set.getStorageSetId()
def unpackXmlDoc(self, xmlDoc):
"""
Unpack the configuration file represented as the XML ASCII text.
xmlDoc: XML ASCII document (string).
Returns: Reference to object itself.
"""
domObj = None
try:
domObj = xml.dom.minidom.parseString(xmlDoc)
except Exception, e:
if (domObj != None): domObj.unlink()
ex = str(e)
lineNo = str(e).split(":")[1]
errMsg = "Error parsing NG/AMS XML Configuration. " +\
"Probably around line number: " + str(lineNo) + ". " +\
"Exception: " + str(e)
errMsg = genLog("NGAMS_ER_CONF_FILE", [errMsg])
raise Exception, errMsg
def read(self, dbConObj, diskId):
"""
Query information about a specific HDD and set the
class member variables accordingly.
dbConObj: Reference to DB connection object (ngamsDb).
diskId: Disk ID (string).
Returns: Reference to object itself.
"""
T = TRACE()
res = dbConObj.getDiskInfoFromDiskId(diskId)
if (res == []):
errMsg = genLog("NGAMS_ER_UNKNOWN_DISK", [diskId])
raise Exception, errMsg
else:
self.unpackSqlResult(res)
return self
def dumpDiskInfo(hostId, dbConObj, ngamsCfgObj, diskId, mountPoint):
"""
Dump the disk status information for one disk.
hostId: The ID of this NGAS host
dbConObj: DB connection object (ngamsDb).
ngamsCfgObj: NG/AMS Configuration Object (ngamsConfig).
diskId: Disk ID (string).
mountPoint: Mount point (string).
Returns: NgasDiskInfo XML document (string/xml).
"""
# Get disk info from DB
diskInfo = ngamsDiskInfo.ngamsDiskInfo()
try:
diskInfo.read(dbConObj, diskId)
except:
errMsg = genLog("NGAMS_ER_DISK_STATUS", [diskId])
logger.error(errMsg)
ngamsNotification.notify(hostId, ngamsCfgObj, NGAMS_NOTIF_ERROR,
"MISSING DISK IN DB", errMsg)
return
# Create status object and save the XML in the status file
ngasDiskInfo = prepNgasDiskInfoFile(hostId, diskInfo)
filename = os.path.normpath(mountPoint + "/" + NGAMS_DISK_INFO)
# Check if it is possible to write in the file if it exists.
if (os.path.exists(filename) and (not ngamsLib.fileWritable(filename))):
return
logger.debug("Writing NGAS Disk Status for disk with ID: %s into file: %s",
diskId, filename)
fd = open(filename, "w")
fd.write(ngasDiskInfo)
fd.close()
return ngasDiskInfo
def detMimeType(mimeTypeMaps,
filename,
noException = 0):
"""
Determine mime-type of a file, based on the information in the
NG/AMS Configuration and the filename (extension) of the file.
mimeTypeMaps: See ngamsConfig.getMimeTypeMappings() (list).
filename: Filename (string).
noException: If the function should not throw an exception
this parameter should be 1. In that case it
will return NGAMS_UNKNOWN_MT (integer).
Returns: Mime-type (string).
"""
# Check if the extension as ".<ext>" is found in the filename as
# the last part of the filename.
logger.debug("Determining mime-type for file with URI: %s ...", filename)
found = 0
mimeType = ""
for map in mimeTypeMaps:
ext = "." + map[1]
idx = string.find(filename, ext)
if ((idx != -1) and ((idx + len(ext)) == len(filename))):
found = 1
mimeType = map[0]
break
if ((not found) and noException):
return NGAMS_UNKNOWN_MT
elif (not found):
errMsg = genLog("NGAMS_ER_UNKNOWN_MIME_TYPE1", [filename])
raise Exception, errMsg
else:
logger.debug("Mime-type of file with URI: %s determined: %s",
filename, mimeType)
return mimeType
def sendEmail(ngamsCfgObj,
smtpHost,
subject,
recList,
fromField,
dataRef,
contentType=None,
attachmentName=None,
dataInFile=0):
"""
Send an e-mail to the recipients specified.
ngamsCfgObj: Reference to object containing NG/AMS
configuration file (ngamsConfig).
smtpHost: Mail server to use for sending the mail (string).
subject: Subject of mail message (string).
recList: List containing recipients, e.g. [email protected] (string).
fromField: Name for the from field (string).
dataRef: Message to send or reference to file containing data
to send (string).
contentType: Mime-type of message (string).
attachmentName: Name of attachment in mail (string).
dataInFile: Used to make the sendmail tool send the data contained
in a file (integer).
Returns: Void.
"""
T = TRACE()
hdr = "Subject: " + subject + "\n"
if (contentType):
hdr += "Content-Type: " + contentType + "\n"
if (attachmentName):
hdr += "Content-Disposition: attachment; filename=" +\
attachmentName + "\n"
if (not dataInFile):
data = hdr + "\n" + dataRef
else:
# Prepare a file with the exact contents of the email.
data = genTmpFilename(ngamsCfgObj, "_NOTIF_EMAIL.tmp")
fo = open(data, "w")
fo.write(hdr + "\n")
foIn = open(dataRef)
while (1):
buf = foIn.read(16384)
if (buf == ""): break
fo.write(buf)
fo.close()
foIn.close()
for emailAdr in recList:
try:
server = ngamsSmtpLib.ngamsSMTP(smtpHost)
server.sendMail("From: " + fromField, "Bcc: " + emailAdr, data, [],
[], dataInFile)
except Exception, e:
if (dataInFile): rmFile(data)
errMsg = genLog("NGAMS_ER_EMAIL_NOTIF",
[emailAdr, fromField, smtpHost,
str(e)])
raise Exception(errMsg)
def checkDisks(hostId, dbConObj, ngamsCfgObj, diskDic):
"""
First it is checked if all HDDs marked in the DB as mounted in this
NGAS system are actually mounted.
Check the HDDs currently installed and for each HDD it is checked
if there is an entry for this. If not a new entry for this HDD is
created.
Subsequently it is checked if each of the disks now marked as mounted
and thus available in this system, are really available. If not,
an exception is thrown.
Finally it is checked that for each Stream defined in the NG/AMS
Configuration, that the disks of at least one Storage Set are
available.
hostId: The ID of this NGAS host
dbConObj: DB connection object (ngamsDb).
ngamsCfgObj: Configuration object (ngamsConfig).
diskDic: Dictionary containing ngamsPhysDiskInfo objects with the
information about the disk configuration (dictionary).
Returns: Void.
"""
T = TRACE()
diskList = getDiskInfoForMountedDisks(dbConObj, hostId,
ngamsCfgObj.getRootDirectory())
# The "mtDiskDic" dictionary contains information (ngasDiskInfo objects)
# for each disk mounted on this system.
if (diskList != []):
logger.debug("All System Types: Checking if disks registered in the " +\
"DB as mounted on this system really are mounted physically ...")
mtDiskDic = {}
for disk in diskList:
logger.info("Checking for availability of disk with ID: %s",
disk.getDiskId())
found = 0
for slotId in diskDic.keys():
if (diskDic[slotId].getDiskId() == disk.getDiskId()):
found = 1
mtDiskDic[slotId] = disk
break
if (found == 0):
logger.info(
"Disk with ID: %s not available anymore - modifying entry in DB.",
disk.getDiskId())
disk.setHostId("")
disk.setSlotId("")
disk.setMountPoint("")
disk.setMounted(0)
disk.write(dbConObj)
else:
logger.info("Disk with ID: %s available.", disk.getDiskId())
# Ensure we have the information available for each disk, either from the
# DB or from the NGAS Disk Info files (if one of these are available).
#
# Note: If both information about the disk is available in the DB and
# in the NgasDiskInfo file, the information for the disk is taken
# from the one, which has the most recent InstallationDate defined.
slotIdList = []
for slotId in diskDic.keys():
slotIdList.append(slotId)
slotIdList.sort()
for slotId in slotIdList:
slotId = str(slotId)
# Get from DB if not already read + if disk in DB.
if (not mtDiskDic.has_key(slotId)):
if (dbConObj.diskInDb(diskDic[slotId].getDiskId())):
dbDiskInfo = ngamsDiskInfo.ngamsDiskInfo()
dbDiskInfo.read(dbConObj, diskDic[slotId].getDiskId())
mtDiskDic[slotId] = dbDiskInfo
else:
dbDiskInfo = mtDiskDic[slotId]
# Get info from NgasDiskInfo file (if available).
ngasDiskInfoFile = getNgasDiskInfoFile(diskDic, slotId)
# Decide which information to use.
if (not mtDiskDic.has_key(slotId)):
mtDiskDic[slotId] = ngasDiskInfoFile
elif (mtDiskDic.has_key(slotId) and ngasDiskInfoFile):
# Take the info in the NgasDiskInfo file if this is the most
# recent one.
if (ngasDiskInfoFile.getInstallationDate() >
dbDiskInfo.getInstallationDate()):
logger.info("Disk information in NgasDiskInfo File for disk with "+\
"ID: %s / Slot ID: %s, is more recent than the one found in the " +\
"NGAS DB for that disk. Taking information for disk " +\
"from NgasDiskInfo File.",
ngasDiskInfoFile.getDiskId(), slotId)
mtDiskDic[slotId] = ngasDiskInfoFile
else:
mtDiskDic[slotId] = None
if (ngamsCfgObj.getAllowArchiveReq() and (diskDic != {})):
logger.info("Archiving System: Checking that all disks defined in the " +\
"configuration and which are mounted and not completed, can be "+\
"accessed. Check that installed disks, have entries in the " +\
"NGAS DB ...")
mtSlotIds = diskDic.keys()
cfgSlotIds = []
for cfgSlotId in ngamsCfgObj.getSlotIds():
cfgSlotIds.append(cfgSlotId)
cfgSlotIds.sort()
notifInfo = []
# Disks, which should not be updated/inserted in the DB are contained
# in the "rmDiskDic" dictionary.
rmDiskDic = {}
# Disks, which should be updated/inserted in the DB are contained
# in the "dbUpdateDic" dictionary.
dbUpdateDic = {}
for slotId in cfgSlotIds:
slotId = str(slotId)
if slotId in mtSlotIds:
diskInfo = diskDic[slotId]
assocSlotId = ngamsCfgObj.getAssocSlotId(slotId)
# If a disk is marked as 'completed' in the DB or in the
# NGAS Disk Info File, we simply update its entry in the DB.
if (mtDiskDic[slotId]):
if (mtDiskDic[slotId].getCompleted()):
logger.info("Disk in Slot with ID: %s " +\
"is marked as 'completed' - just updating " +\
"info in DB", slotId)
dbUpdateDic[slotId] = ["UPDATE", mtDiskDic[slotId]]
continue
# Check for disk accessibility
diskRef = "Slot ID: " + str(diskInfo.getSlotId()) +\
" - Disk ID: " + diskInfo.getDiskId()
logger.info("Checking for accessibility of disk: %s", diskRef)
if (checkDiskAccessibility(diskInfo.getMountPoint()) == -1):
errMsg = genLog("NGAMS_ER_DISK_INACCESSIBLE", [diskRef])
logger.error(errMsg)
rmDiskDic[slotId] = slotId
notifInfo.append(["DISK INACCESSIBLE", errMsg])
if (dbUpdateDic.has_key(assocSlotId)):
dbUpdateDic[assocSlotId] = None
del dbUpdateDic[assocSlotId]
continue
else:
logger.info("Disk: %s is accessible", diskRef)
logger.info(
"Check if the disk with ID: %s already has an entry in the DB",
diskInfo.getDiskId())
tmpDiskInfo = mtDiskDic[slotId]
# Check the disk information or simply mark the disk
# for update in the DB.
if (tmpDiskInfo == None):
# If the associated disk is not already found to be
# wrong, we add an ADD entry in the Update DB Dictionary.
if (not rmDiskDic.has_key(assocSlotId)):
dbUpdateDic[slotId] = ["ADD"]
else:
logger.info("Entry found in DB for disk with ID: %s",
diskInfo.getDiskId())
logger.info("Check that this disk already registered " +\
"is used correctly as Main or Replication Disk ...")
# From the Logical Name of the disk, we can identify
# if it was previously used as a Main or as a Replication
# Disk (e.g.: "LS-FitsStorage3-M-000003"). From the Slot
# ID we check via the NG/AMS Configuration if the disk is
# installed in a Main or Replication Slot.
try:
tmpType = tmpDiskInfo.getLogicalName().split("-")[-2]
except Exception, e:
raise Exception, "Illegal Logical Name specified: " +\
str(tmpDiskInfo.getLogicalName()) + " for " +\
"disk with ID: " + tmpDiskInfo.getDiskId()
if (tmpType == "M"):
prevMainDisk = 1
else:
prevMainDisk = 0
mainDisk = isMainDisk(slotId, ngamsCfgObj)
# Check: Disk was previously installed as a Main Disk,
# but is now installed in a Replication Slot.
if (prevMainDisk and not mainDisk):
errMsg = genLog(
"NGAMS_ER_MAIN_DISK_WRONGLY_USED",
[str(slotId),
tmpDiskInfo.getLogicalName()])
logger.error(errMsg)
notifInfo.append(["MAIN DISK USED AS " +\
"REPLICATION DISK", errMsg])
rmDiskDic[slotId] = slotId
continue
# Check: Disk was previously installed as a Replication
# Disk, but is now installed in a Main Slot.
if (not prevMainDisk and mainDisk):
errMsg = genLog(
"NGAMS_ER_REP_DISK_WRONGLY_USED",
[str(slotId),
tmpDiskInfo.getLogicalName()])
logger.error(errMsg)
notifInfo.append(["REPLICATION DISK USED AS " +\
"MAIN DISK", errMsg])
rmDiskDic[slotId] = slotId
continue
# Everything OK, update existing entry in the DB.
# If the associated disk is not already found to be
# wrong, we add an entry in the Update DB Dictionary.
if (not rmDiskDic.has_key(assocSlotId)):
dbUpdateDic[slotId] = ["UPDATE", tmpDiskInfo]
# Create some standard directories on the disk if not
# already there.
if (ngamsCfgObj.getAllowArchiveReq()
or ngamsCfgObj.getAllowRemoveReq()):
diskDbDir = os.path.normpath(diskInfo.getMountPoint() +\
"/" + NGAMS_DB_DIR)
checkCreatePath(diskDbDir)
diskDbCacheDir = os.path.\
normpath(diskInfo.getMountPoint() +\
"/" + NGAMS_DB_CH_CACHE)
checkCreatePath(diskDbCacheDir)
# Check that each disk accepted so far has an associated disk if this
# is specified like this in the configuration.
logger.info("Check if each disk has an associated disk if the " +\
"configuration specifies this ...")
for slotId in dbUpdateDic.keys():
# We do not consider disks that are already completed.
if (mtDiskDic[slotId]):
if (mtDiskDic[slotId].getCompleted()): continue
assocSlotId = ngamsCfgObj.getAssocSlotId(slotId)
if (assocSlotId):
if (not dbUpdateDic.has_key(assocSlotId)):
msg = "Disk in slot: %s has no associated " +\
"disk in slot: %s - rejecting disk in slot: %s"
logger.info(msg, slotId, assocSlotId, slotId)
del dbUpdateDic[slotId]
rmDiskDic[slotId] = slotId
# Remove entries from the Disk Dictionary, which are invalid.
for slId in rmDiskDic.keys():
for id in [slId, ngamsCfgObj.getAssocSlotId(slId)]:
if (diskDic.has_key(id)):
logger.info("Removing invalid disk info object from Disk " +\
"Dictionary - Port No: %s Mount Point: %s Disk ID: %s",
str(diskDic[id].getPortNo()),
str(diskDic[id].getMountPoint()),
diskDic[id].getDiskId())
diskDic[id] = None
del diskDic[id]
if (dbUpdateDic.has_key(id)):
dbUpdateDic[id] = None
del dbUpdateDic[id]
# Generate a Notification Message with the possible errors
# accumulated so far and send out Error Notification Messages.
if (len(notifInfo) > 0):
errMsg = "FOLLOWING PROBLEMS WERE ENCOUNTERED WHILE CHECKING " +\
"THE NGAS DISK CONFIGURATION:\n"
for err in notifInfo:
errMsg = errMsg + "\n" + err[0] + ":\n" + err[1] + "\n"
ngamsNotification.notify(hostId, ngamsCfgObj, NGAMS_NOTIF_ERROR,
"DISK CONFIGURATION INCONSISTENCIES/" +\
"PROBLEMS ENCOUNTERED", errMsg)
# Write information in the NGAS DB about the disks available.
slotIds = []
for slotId in dbUpdateDic.keys():
slotIds.append(slotId)
# Sort the slots such that the Main Slot is always listed before the
# Replication Slot + sort the slots 'logically' according to the name.
slotIdDic = {}
for slotId in slotIds:
storageSet = ngamsCfgObj.getStorageSetFromSlotId(slotId)
slotIdDic[storageSet.getMainDiskSlotId()] = storageSet
tmpSortedSlotIds = ngamsLib.logicalStrListSort(slotIdDic.keys())
sortedSlotIds = []
for slotId in tmpSortedSlotIds:
slotId = slotId.strip()
storageSet = slotIdDic[slotId]
sortedSlotIds.append(storageSet.getMainDiskSlotId())
if (storageSet.getRepDiskSlotId() != ""):
sortedSlotIds.append(storageSet.getRepDiskSlotId())
# Add or update the information for the Storage Sets in the DB
for slId in sortedSlotIds:
if (dbUpdateDic[slId][0] == "ADD"):
addDiskInDb(hostId, dbConObj, ngamsCfgObj, slId, diskDic)
else:
updateDiskInDb(hostId, dbConObj, ngamsCfgObj, slId, diskDic,
dbUpdateDic[slId][1])
logger.info("Archiving System: Check that there is at least one " +\
"Storage Set available for each Stream defined ...")
probMimeTypeList = []
for stream in ngamsCfgObj.getStreamList():
logger.info(
"Checking for target disks availability for mime-type: %s",
stream.getMimeType())
if (checkStorageSetAvailability(hostId, dbConObj, ngamsCfgObj,
stream.getMimeType()) !=
NGAMS_SUCCESS):
probMimeTypeList.append(stream.getMimeType())
if ((len(probMimeTypeList) > 0)
and (not ngamsCfgObj.getArchiveUnits())):
errMsg = ""
for mimeType in probMimeTypeList:
errMsg = errMsg + " " + mimeType
errMsg = genLog("NGAMS_WA_NO_TARG_DISKS", [errMsg])
logger.warning(errMsg)
ngamsNotification.notify(hostId, ngamsCfgObj, NGAMS_NOTIF_NO_DISKS,
"DISK SPACE INAVAILABILITY", errMsg)
ngamsHostInfo objects (dictionary).
"""
T = TRACE()
try:
hostInfoDic = getHostInfoFromHostIds(dbConObj, hostList)
except Exception, e:
hostInfoDic = {}
for host in hostList:
hostInfoDic[host] = None
if (not hostInfoDic.has_key(localHostId)):
_addHostInDic(dbConObj, localHostId, hostInfoDic)
for hostName in hostList:
if (not hostInfoDic.has_key(hostName)):
errMsg = genLog("NGAMS_AL_MIS_HOST", [hostName])
raise Exception, errMsg
hi = hostInfoDic[hostName]
# Find out if this host is local, within a cluster, within the same
# domain or remote.
hostIpInfo = hi.getIpAddress().split(".")
if (hi.getHostId() == localHostId):
hi.setHostType(NGAMS_HOST_LOCAL)
elif (hi.getClusterName() == hostInfoDic[localHostId].getClusterName()
):
# Host is within the same cluster as where this request is handled.
hi.setHostType(NGAMS_HOST_CLUSTER)
elif (hi.getDomain() == hostInfoDic[localHostId].getDomain()):
# Host is within the same domain. Set the information about the
# host to be contacted for handling the request.
if (domObj != None): domObj.unlink()
ex = str(e)
lineNo = str(e).split(":")[1]
errMsg = "Error parsing NG/AMS XML Configuration. " +\
"Probably around line number: " + str(lineNo) + ". " +\
"Exception: " + str(e)
errMsg = genLog("NGAMS_ER_CONF_FILE", [errMsg])
raise Exception, errMsg
# Check that the root element is present.
rootElName = self.__rootElObj.getName()
nodeList = domObj.getElementsByTagName(rootElName)
if (len(nodeList) == 0):
msg = "XML document, does not have the " +\
"proper root element: %s! Aborting."
errMsg = genLog("NGAMS_ER_CONF_FILE", [msg % rootElName])
raise Exception, errMsg
try:
self.clear()
curId = None
self._unpack(nodeList[0], "", None, curId)
domObj.unlink()
except Exception, e:
if (domObj): domObj.unlink()
raise e
return self
def _unpack(self, nodeObj, elDicKey, parElObj, curId):
"""
Unpack configuration XML document starting from the node of the DOM.
def unpackHttpInfo(self, ngamsCfgObj, httpMethod, path, headers):
"""
Unpack the information from an HTTP request and set the
members of the class accordingly.
httpMethod: HTTP request method (GET/POST) (string).
path: Path of HTTP request (string).
headers: Dictionary containing the information for the
headers of the HTTP query (dictionary).
Returns: Reference to object itself.
"""
T = TRACE()
self.setHttpMethod(httpMethod)
# Handle the HTTP headers.
for key in headers.keys():
keyTmp = key.lower()
val = urllib.unquote(headers[key])
logger.debug("Parsing HTTP header key: %s with value: %s", key,
val)
self.__httpHdrDic[key] = val
if (keyTmp == "content-disposition"):
pars = ngamsLib.parseHttpHdr(headers[key])
for name, val in pars.items():
val = urllib.unquote(val)
if name == "filename":
self.setFileUri(os.path.basename(val))
elif name == "mime_type":
if (self.getMimeType() == ""):
self.setMimeType(val)
elif name.strip():
self.addHttpPar(name, val)
elif (keyTmp == "content-type"):
if (self.getMimeType() == ""):
self.setMimeType(val.split(";")[0].strip(" \""))
elif (keyTmp == "content-length"):
self.setSize(val.strip(" \""))
elif (keyTmp == "authorization"):
self.setAuthorization(urllib.unquote(val.strip()))
# Handle the information in the path.
path, query = urllib.splitquery(path)
self.setCmd(path.lstrip('/'))
if (query):
parList = urlparse.parse_qsl(query)
for name, val in parList:
logger.debug("Found parameter: %s with value: %s", name, val)
if (httpMethod
in [NGAMS_HTTP_GET, NGAMS_HTTP_PUT, NGAMS_HTTP_POST]):
self.addHttpPar(name, val)
# Subscription file delivery is always POST, but sometimes we want it behave like GET (e.g. proxy qrchive) to pass on parametres in url string.
if name == "filename":
self.setFileUri(val)
elif name == "mime_type":
self.setMimeType(val)
elif name == "authorization":
self.setAuthorization(val)
# Small trick to set the mime-type in case not defined by the
# Content-Type HTTP header.
if ((self.getCmd() == NGAMS_ARCHIVE_CMD)
and ((self.getMimeType() == "") or
((self.getMimeType() == NGAMS_ARCH_REQ_MT)))):
if (self.getFileUri().strip() == ""):
raise Exception, genLog(
"NGAMS_ER_CMD_EXEC",
[NGAMS_ARCHIVE_CMD, "Missing parameter: filename"])
mimeType = ngamsLib.detMimeType(ngamsCfgObj.getMimeTypeMappings(),
self.getFileUri(), 1)
if (mimeType == NGAMS_UNKNOWN_MT):
errMsg = genLog("NGAMS_ER_UNKNOWN_MIME_TYPE1",
[self.getFileUri()])
raise Exception, errMsg
else:
self.setMimeType(mimeType)
# In the case of an Archive Request, check that the mime-type is
# known to this NGAS installation.
if (self.getCmd() == NGAMS_ARCHIVE_CMD):
# - To do this, we check if there is a Stream defined for
# this kind of data.
if (not ngamsCfgObj.getStreamFromMimeType(self.getMimeType())):
errMsg = genLog(
"NGAMS_ER_UNKNOWN_MIME_TYPE2",
[self.getMimeType(), self.getFileUri()])
raise Exception, errMsg
return self
def findTargetDisk(hostId,
dbConObj,
ngamsCfgObj,
mimeType,
sendNotification=1,
diskExemptList=[],
caching=0,
reqSpace=None):
"""
Find a target disk for a file being received.
dbConObj: DB connection object (ngamsDb).
ngamsCfgObj: Instance of NG/AMS Configuration Class (ngamsConfig).
mimeType: Mime-type of file (string).
sendNotification: 1 = send Email Notification Message (integer).
diskExemptList: List with Disk IDs of disks, which it is not
desirable to consider (list/string).
caching: Used to increase the speed of this function
if it is invoked many times sequentially. SHOULD
BE USED WITH GREAT CARE! (integer/0|1).
reqSpace: The required space needed in bytes (integer).
Returns: ngamsDiskInfo object containing the necessary
information (ngamsDiskInfo).
"""
T = TRACE()
startTime = time.time()
# Get Disk IDs matching the mime-type.
logger.debug("Finding target disks - mime-type is: %s", mimeType)
global _diskInfoObjsDic
if ((not caching) or (not _diskInfoObjsDic.has_key(mimeType))):
diskInfoObjs = getDiskInfoObjsFromMimeType(hostId, dbConObj,
ngamsCfgObj, mimeType,
sendNotification)
if (caching): _diskInfoObjsDic[mimeType] = diskInfoObjs
else:
diskInfoObjs = _diskInfoObjsDic[mimeType]
# Analyze which of the dynamic Disks Sets are available to be used
# as Target Disk Set, i.e., only Disk Sets where both Main Disk and
# Replication Disk (if available) are not completed.
diskIdDic = {}
slotIdDic = {}
for diskInfoObj in diskInfoObjs:
diskIdDic[diskInfoObj.getDiskId()] = diskInfoObj
slotIdDic[diskInfoObj.getSlotId()] = diskInfoObj
# If the disk is a Main Disk, and if it is not completed, and if there is
# an associated Replication Disk and it is also not completed, we accept
# the disk as a possible candidate disk.
diskIds = []
for diskInfoObj in diskInfoObjs:
# We disregard disks listed in the exemptlist.
if diskInfoObj.getDiskId() in diskExemptList:
continue
# Only a Main Disk (not completed) can be considered.
if (isMainDisk(diskInfoObj.getSlotId(), ngamsCfgObj)
and (not diskInfoObj.getCompleted())):
# Check if the Main Disk has enough space on it to store the file.
# Even though a disk is not completed, it may be that there is not
# enough space to store a given file.
if (reqSpace):
if (diskInfoObj.getAvailableMb() < (reqSpace / 1048576.)):
continue
# If replication is on, check the associated Replication Disk
# (if any).
repSlotId = ngamsCfgObj.getAssocSlotId(diskInfoObj.getSlotId())
if (ngamsCfgObj.getReplication() and (repSlotId != "")
and (slotIdDic.has_key(repSlotId))):
repDiskObj = slotIdDic[repSlotId]
if (not repDiskObj.getCompleted()):
# Check if the Replication Disk has enough space.
if (reqSpace):
if (repDiskObj.getAvailableMb() <
(reqSpace / 1048576.)):
continue
diskIds.append(diskInfoObj.getDiskId())
else:
diskIds.append(diskInfoObj.getDiskId())
# If no storage sets found, generate a log message.
if (diskIds == []):
errMsg = genLog("NGAMS_AL_NO_STO_SETS", [mimeType])
logger.warning(errMsg)
if (sendNotification):
ngamsNotification.notify(hostId, ngamsCfgObj, NGAMS_NOTIF_NO_DISKS,
"NO DISKS AVAILABLE", errMsg)
raise Exception, errMsg
# Find the best target disk.
global _bestTargetDiskDic
key = str(diskIds)[1:-1].replace("'", "_").replace(", ", "")
if ((not caching) or (not _bestTargetDiskDic.has_key(key))):
diskId = dbConObj.getBestTargetDisk(diskIds,
ngamsCfgObj.getRootDirectory())
if (caching): _bestTargetDiskDic[key] = diskId
else:
diskId = _bestTargetDiskDic[key]
if (diskId == None):
errMsg = genLog("NGAMS_AL_NO_STO_SETS", [mimeType])
logger.warning(errMsg)
if (sendNotification):
ngamsNotification.notify(hostId, ngamsCfgObj, NGAMS_NOTIF_NO_DISKS,
"NO DISKS AVAILABLE", errMsg)
raise Exception, errMsg
else:
global _diskInfoDic
key = diskId + "_" + mimeType
if ((not caching) or (not _diskInfoDic.has_key(key))):
diskInfo = ngamsDiskInfo.ngamsDiskInfo()
diskInfo.getInfo(dbConObj, ngamsCfgObj, diskId, mimeType)
if (caching): _diskInfoDic[key] = diskInfo
else:
diskInfo = _diskInfoDic[key]
return diskInfo
