def genSqlResult(self):
"""
Generate a list with the information as read from ngas_files and
defined by the variables ngamsDbCore.NGAS_FILES_DISK_ID, ... .
Returns: List with information from one row of ngas_files (list).
"""
ing_date = ''
creation_date = ''
if self.__ingestionDate is not None:
ing_date = toiso8601(self.__ingestionDate, local=True)
if self.__creationDate is not None:
creation_date = toiso8601(self.__creationDate, local=True)
return [
self.getDiskId(),
self.getFilename(),
self.getFileId(),
int(self.getFileVersion()),
self.getFormat(),
int(self.getFileSize()),
int(self.getUncompressedFileSize()),
self.getCompression(), ing_date,
int(self.getIgnore()),
self.getChecksum(),
self.getChecksumPlugIn(),
self.getFileStatus(), creation_date,
self.getIoTime(),
self.getIngestionRate(),
self.getContainerId()
]
def getObjStatus(self):
"""
Return a list with the current status of the object. The format
of the list is:
[[<xml attribute name>, <value>, ...], ...]
Returns: List with object status (list/list).
"""
compDate = "" if self.__completionDate is None else toiso8601(
self.__completionDate)
instDate = "" if self.__installationDate is None else toiso8601(
self.__installationDate)
lastCheck = "" if self.__lastCheck is None else toiso8601(
self.__lastCheck)
return [["DiskId", self.getDiskId()], ["Archive",
self.getArchive()],
["InstallationDate", instDate], ["Type",
self.getType()],
["Manufacturer", self.getManufacturer()],
["LogicalName", self.getLogicalName()],
["HostId", self.getHostId()], ["SlotId",
self.getSlotId()],
["Mounted", self.getMounted()],
["MountPoint", self.getMountPoint()],
["NumberOfFiles", self.getNumberOfFiles()],
["AvailableMb", self.getAvailableMb()],
["BytesStored", self.getBytesStoredStr()],
["Completed", self.getCompleted()],
["CompletionDate", compDate], ["Checksum",
self.getChecksum()],
["TotalDiskWriteTime",
self.getTotalDiskWriteTime()], ["LastCheck", lastCheck]]
def prepNgasDiskInfoFile(hostId, diskInfoObj, yankNewlines=0, yankElSpaces=0):
"""
Create and prepare an NGAS Disk Info Status Object and set the
attributes in the ngamsStatus object accordingly. The NgasDiskInfo
XML document is returned.
diskInfoObj: NGAS Disk Info Object containing information about the
disk for which to generate the NgasDiskInfo file
(ngasDiskInfo).
yankNewlines: If set to 1 newline characters will be removed from
the resulting XML document (integer/0|1).
yankElSpaces: Yank spaces between elements (integer/0|1).
Returns: NgasDiskInfo XML document (string/xml).
"""
status = ngamsStatus.ngamsStatus()
status.\
setDate(toiso8601()).\
setVersion(getNgamsVersion()).setHostId(hostId).\
setMessage("Disk status file").addDiskStatus(diskInfoObj)
xmlDoc = status.genXml(0, 1, 0, 0, 1).toprettyxml()
if (yankNewlines): xmlDoc = re.sub("\n", "", str(xmlDoc)).strip()
if (yankElSpaces):
xmlDoc = re.sub("> *<", "><", xmlDoc)
xmlDoc = re.sub(">\t*<", "><", xmlDoc)
return xmlDoc
def convertTimeStamp(self, t):
"""
Convert a timestamp given in one of the following formats:
"""
if isinstance(t, basestring):
return t
else:
return toiso8601(t, local=True)
def getObjStatus(self):
"""
Return a list with the current status of the object. The format
of the list is:
[[<xml attribute name>, <value>, ...], ...]
Returns: List with object status (list/list).
"""
# Fields in XML document/ASCII dump
ing_date = ''
creation_date = ''
mod_date = ''
acc_date = ''
if self.__ingestionDate is not None:
ing_date = toiso8601(self.__ingestionDate)
if self.__creationDate is not None:
creation_date = toiso8601(self.__creationDate)
if self.__modDate is not None:
mod_date = toiso8601(self.__modDate)
if self.__accDate is not None:
acc_date = toiso8601(self.__accDate)
return [["DiskId", self.getDiskId()], ["FileName",
self.getFilename()],
["FileId", self.getFileId()],
["FileVersion", self.getFileVersion()],
["Format", self.getFormat()], ["FileSize",
self.getFileSize()],
["UncompressedFileSize",
self.getUncompressedFileSize()],
["Compression", self.getCompression()],
["IngestionDate", ing_date], ["Ignore",
self.getIgnore()],
["Checksum", self.getChecksum()],
["ChecksumPlugIn", self.getChecksumPlugIn()],
["FileStatus", self.getFileStatus()],
["CreationDate", creation_date], ["Tag", self.getTag()],
["Permissions", self.getPermissions()],
["Owner", self.getOwner()], ["Group", self.getGroup()],
["ModificationDate", mod_date], ["AccessDate", acc_date],
["TotalIoTime", self.getIoTime()],
["IngestionRate", self.getIngestionRate()],
["ContainerId", self.getContainerId()]]
def _dummy_stat(host_id, status, msg, data=None):
stat = ngamsStatus().\
setDate(toiso8601()).\
setVersion(getNgamsVersion()).setHostId(host_id).\
setStatus(status).\
setMessage(msg).\
setState(NGAMS_ONLINE_STATE).\
setSubState(NGAMS_IDLE_SUBSTATE)
if data:
stat.setData(data)
return stat
def getObjStatus(self):
"""
Return a list with the current status of the object. The format
of the list is:
[[<xml attribute name>, <value>, ...], ...]
Returns: List with object status (list/list).
"""
# Fields in XML document/ASCII dump
wakeup_time = ""
inst_date = ""
if self.__srvReqWakeUpTime is not None:
wakeup_time = toiso8601(self.__srvReqWakeUpTime)
if self.__installationDate is not None:
inst_date = toiso8601(self.__installationDate)
return [["Host ID", self.getHostId()], ["Domain",
self.getDomain()],
["IP Address", self.getIpAddress()],
["MAC Address", self.getMacAddress()],
["Number of Slots", self.getNSlots()],
["Cluster Name", self.getClusterName()],
["Installation Date", inst_date],
["Server Version", self.getSrvVersion()],
["Server Port", self.getSrvPort()],
["Server Allow Archiving",
self.getSrvArchive()],
["Server Allow Retrieving",
self.getSrvRetrieve()],
["Server Allow Processing",
self.getSrvProcess()],
["Server Allow Remove",
self.getSrvRemove()],
["Server Data Checking",
self.getSrvDataChecking()],
["Server State", self.getSrvState()],
["Server Suspended",
self.getSrvSuspended()],
["Server Wake-Up Request",
self.getSrvReqWakeUpSrv()],
["Server Wake-Up Time", wakeup_time]]
def dumpBuf(self):
"""
Dump contents of object into a string buffer.
Returns: String buffer with disk info (string).
"""
format = prFormat1()
buf = "Subscriber Info:\n"
buf += format % ("Host ID:", self.getHostId())
if (self.getPortNo() > 0):
buf += format % ("Port Number:", str(self.getPortNo()))
buf += format % ("Priority:", str(self.getPriority()))
buf += format % ("Subscriber ID:", self.getId())
buf += format % ("Subscriber URL:", self.getUrl())
if self.__startDate is not None:
buf += format % ("Start Date:", toiso8601(self.__startDate))
buf += format % ("Filter Plug-In:", self.getFilterPi())
buf += format % ("Filter Plug-In Par.s:", self.getFilterPiPars())
if self.__lastFileIngDate is not None:
buf += format % ("Last File Ing. Date:",
toiso8601(self.__lastFileIngDate))
return buf
def dump(self):
"""
Create an ASCII dump of the contents in a string buffer and return
a reference to the buffer.
Returns: String buffer containing the ASCII dump (string).
"""
T = TRACE()
buf = "Contents of ngamsMirroringRequest:%s:\n" % str(self)
buf += "Instance ID: %s\n" % self.getInstanceId()
buf += "File ID: %s\n" % self.getFileId()
buf += "File Version: %d\n" % self.getFileVersion()
if self.__ingestionDate is not None:
buf += "Ingestion Date: %s\n" % toiso8601(
self.__ingestionDate)
buf += "Server List ID: %d\n" % self.getSrvListId()
buf += "XML File Info:\n%s\n" % self.getXmlFileInfo()
buf += "Status: %s\n" % self.getStatusAsStr()
buf += "Message: %s\n" % str(self.getMessage())
buf += "Last Activity Date: %s\n" % toiso8601(
self.getLastActivityTime())
return buf
def _genXml(self, doc):
contEl = doc.createElement('Container')
contEl.setAttribute('id', str(
self.getContainerId())) # might be an uuid.uuid4 object
contEl.setAttribute('name', self.getContainerName())
contEl.setAttribute('size', str(self.getContainerSize()))
if self._ingestionDate is not None:
contEl.setAttribute('ingestionDate',
toiso8601(self._ingestionDate))
for fileInfo in self._filesInfo:
contEl.appendChild(fileInfo.genXml())
for childCont in self._containers:
contEl.appendChild(childCont._genXml(doc))
return contEl
def genSummary(self):
"""
Generate a summary of the contents of the object in one line.
Returns: Reference to string buffer with the object summary (string).
"""
T = TRACE(5)
buf = "Summary of Mirroring Request Object: "
buf += "Instance ID: %s" % self.getInstanceId()
buf += ". File ID: %s" % self.getFileId()
buf += ". File Version: %d" % self.getFileVersion()
buf += ". Status: %s" % self.getStatusAsStr()
buf += ". Message: %s" % str(self.getMessage())
buf += ". Last Activity Date: %s" % toiso8601(
self.getLastActivityTime())
return buf
def genXml(self):
"""
Generate an XML DOM Node and with the information in the object
and return this.
Returns: DOM node object (Node).
"""
tmpSubscrEl = xml.dom.minidom.Document().createElement("Subscriber")
tmpSubscrEl.setAttribute("HostId", self.getHostId())
if (self.getPortNo() > 0):
tmpSubscrEl.setAttribute("PortNo", str(self.getPortNo()))
tmpSubscrEl.setAttribute("Priority", str(self.getPriority()))
tmpSubscrEl.setAttribute("SubscriberUrl", self.getUrl())
if self.__startDate is not None:
tmpSubscrEl.setAttribute("StartDate", toiso8601(self.__startDate))
tmpSubscrEl.setAttribute("FilterPlugIn", self.getFilterPi())
tmpSubscrEl.setAttribute("FilterPlugInPars", self.getFilterPiPars())
return tmpSubscrEl
def createDbFileChangeStatusDoc(self,
hostId,
operation,
fileInfoObjList,
diskInfoObjList=[]):
"""
The function creates a pickle document in the '<Disk Mt Pt>/.db/cache'
directory from the information in the 'fileInfoObj' object.
operation: Has to be either ngams.NGAMS_DB_CH_FILE_INSERT or
ngams.NGAMS_DB_CH_FILE_DELETE (string).
fileInfoObj: List of instances of NG/AMS File Info Object
containing the information about the file
(list/ngamsFileInfo).
diskInfoObjList: It is possible to give the information about the
disk(s) in question via a list of ngamsDiskInfo
objects (list/ngamsDiskInfo).
Returns: Void.
"""
T = TRACE()
# TODO: Implementation concern: This class is suppose to be
# at a lower level in the hierarchie than the ngamsFileInfo,
# ngamsFileList, ngamsDiskInfo and ngamsStatus classes and as
# such these should not be used from within this class. All usage
# of these classes in the ngamsDbBase class should be analyzed.
# Probably these classes should be made base classes for this class.
# TODO: Potential memory bottleneck.
timeStamp = toiso8601()
# Sort the File Info Objects according to disks.
fileInfoObjDic = {}
for fileInfo in fileInfoObjList:
if (not fileInfoObjDic.has_key(fileInfo.getDiskId())):
fileInfoObjDic[fileInfo.getDiskId()] = []
fileInfoObjDic[fileInfo.getDiskId()].append(fileInfo)
# Get the mount points for the various disks concerned.
mtPtDic = {}
if (diskInfoObjList == []):
diskIdMtPtList = self.getDiskIdsMtPtsMountedDisks(hostId)
for diskId, mtPt in diskIdMtPtList:
mtPtDic[diskId] = mtPt
else:
for diskInfoObj in diskInfoObjList:
mtPtDic[diskInfoObj.getDiskId()] = diskInfoObj.getMountPoint()
# Create on each disk the relevant DB Change Status Document.
tmpFileList = None
for diskId in fileInfoObjDic.keys():
if ((len(fileInfoObjList) == 1) and (diskInfoObjList == [])):
tmpStatObj = fileInfoObjList[0].setTag(operation)
else:
# Apparently the dbId is used only as the ID of the file list,
# which is totally irrelevant (this object seems to be read on
# ngamsJanitorThread#checkDbChangeCache)
dbId = diskId + "." + timeStamp
tmpFileObjList = fileInfoObjDic[diskId]
tmpFileList = ngamsFileList.ngamsFileList(dbId, operation)
for fileObj in tmpFileObjList:
tmpFileList.addFileInfoObj(fileObj)
tmpStatObj = ngamsStatus.ngamsStatus().\
setDate(timeStamp).\
setVersion(getNgamsVersion()).\
setHostId(hostId).\
setMessage(dbId).\
addFileList(tmpFileList)
two_stage_pickle(mtPtDic[diskId], tmpStatObj)
def genXml(self,
genCfgStatus=0,
genDiskStatus=0,
genFileStatus=0,
genStatesStatus=1,
genLimDiskStatus=0):
"""
Generate an XML Node which contains the status report (root node).
genCfgStatus: 1 = generate configuration status (integer).
genDiskStatus: 1 = generate disk status (integer).
genFileStatus: 1 = generate file status (integer).
genStatesStatus: 1 = generate State/Sub-State (integer).
genLimDiskStatus: 1 = generate only a limited (generic) set
of disk status (integer).
Returns: XML document (xml.dom.minidom.Document).
"""
T = TRACE(5)
doc = xml.dom.minidom.Document()
ngamsStatusEl = doc.createElement(NGAMS_XML_STATUS_ROOT_EL)
doc.appendChild(ngamsStatusEl)
# Status Element.
statusEl = xml.dom.minidom.Document().createElement("Status")
statusEl.setAttribute("Date", self.getDate())
statusEl.setAttribute("Version", self.getVersion())
statusEl.setAttribute("HostId", self.getHostId())
statusEl.setAttribute("Message", self.getMessage())
if (genStatesStatus):
statusEl.setAttribute("Status", self.getStatus())
statusEl.setAttribute("State", self.getState())
statusEl.setAttribute("SubState", self.getSubState())
# Add the request handling status (if defined).
if (self.getRequestId()):
statusEl.setAttribute("RequestId", str(self.getRequestId()))
if self.__requestTime is not None:
statusEl.setAttribute("RequestTime", toiso8601(self.__requestTime))
if (self.getCompletionPercent()):
statusEl.setAttribute("CompletionPercent",
"%.2f" % self.getCompletionPercent())
if (self.getExpectedCount() != None):
statusEl.setAttribute("ExpectedCount",
str(self.getExpectedCount()))
if (self.getActualCount() != None):
statusEl.setAttribute("ActualCount", str(self.getActualCount()))
if self.__estTotalTime is not None:
statusEl.setAttribute("EstTotalTime", str(self.__estTotalTime))
if self.__remainingTime is not None:
statusEl.setAttribute("RemainingTime", str(self.__remainingTime))
if self.__lastRequestStatUpdate is not None:
statusEl.setAttribute("LastRequestStatUpdate",
toiso8601(self.__lastRequestStatUpdate))
if self.__completionTime is not None:
statusEl.setAttribute("CompletionTime",
toiso8601(self.__completionTime))
ngamsStatusEl.appendChild(statusEl)
# NgamsCfg Element.
if (genCfgStatus):
ngamsCfgEl = self.__ngamsCfg.genXml()
ngamsStatusEl.appendChild(ngamsCfgEl)
# DiskStatus Elements.
if (genDiskStatus):
for disk in self.getDiskStatusList():
diskStatusEl = disk.genXml(genLimDiskStatus, genFileStatus)
ngamsStatusEl.appendChild(diskStatusEl)
# FileList Elements.
if (len(self.getFileListList())):
for fileListObj in self.getFileListList():
fileListXmlNode = fileListObj.genXml()
ngamsStatusEl.appendChild(fileListXmlNode)
# Container Elements
for container in self.getContainerList():
ngamsStatusEl.appendChild(container.genXml())
return doc
def notify(hostId,
ngamsCfgObj,
type,
subject,
dataRef,
recList=[],
force=0,
contentType=None,
attachmentName=None,
dataInFile=0):
"""
Send an Notification Email to the subscriber(s) about an event happening.
srvObj: Reference to NGAS server object (ngamsServer).
type: Type of Notification (See NGAMS_NOTIF_* in ngams).
subject: Subject of message (string).
dataRef: Message to send or filename containing data (string).
recList: List with recipients that should receive the Notification
Message (list).
force: Force distribution of Notification Message even though
this was disabled in the configuration (integer/0|1).
contentType: Mime-type of message (string).
attachmentName: Name of attachment in mail (string).
dataInFile: Indicates that data to be send is stored in a file
(integer/0|1).
Returns: Void.
"""
T = TRACE()
if ((not force) or (ngamsCfgObj.getNotifActive() != 1)):
logger.debug("Leaving notify() with no action (disabled/force=0)")
return
# Force emission if data is contained in a file and the (Notification
# Service is activated).
if (dataInFile): force = 1
global notifSem_
try:
# Take Notification Semaphore.
notifSem_.acquire()
# Load a possible pickled Retention Buffer.
pickleObjFile = _getNotifRetBufPickleFile(ngamsCfgObj, hostId)
global retentionBuf_
if (not retentionBuf_):
# Check if pickled Retention Object exists.
if (os.path.exists(pickleObjFile)):
retentionBuf_ = ngamsLib.loadObjPickleFile(pickleObjFile)
else:
retentionBuf_ = {}
# The Retention Buffer is handled the following way:
#
# * If the Retention Buffer does not have an entry for that Message ID,
# create a new entry for that Message ID + store the Last Emission
# Time.
#
# * If the time since the Last Emission Time is less than the Max.
# Retention Time from the configuration, we buffer this message
# in the Retention Buffer.
msgId = type + ":" + subject
timeNow = time.time()
maxRetTime = isoTime2Secs(ngamsCfgObj.getMaxRetentionTime())
maxRetNo = ngamsCfgObj.getMaxRetentionSize()
if ((not force) and (not retentionBuf_.has_key(msgId))):
# We simply create a new entry for this Message ID and initialize
# the Last Emission Time. This message will be sent out.
retentionBuf_[msgId] = [timeNow, None, type, subject, recList, []]
ngamsLib.createObjPickleFile(pickleObjFile, retentionBuf_)
_sendNotifMsg(hostId, ngamsCfgObj, type, subject, dataRef, recList,
contentType, attachmentName)
elif (not force):
logger.debug("Appending Notification Message with ID: %s " + +\
"in Notification Retention Buffer", msgId)
# Set the Retention Start Time if not set.
if (not retentionBuf_[msgId][1]): retentionBuf_[msgId][1] = timeNow
# Append the new element containing the information about the
# message to be retained.
isoTime = toiso8601(timeNow)
retentionBuf_[msgId][5].append([isoTime, dataRef])
# Update the Notification Retention Buffer Pickle File and check
# if there are messages to send out.
ngamsLib.createObjPickleFile(pickleObjFile, retentionBuf_)
_checkSendNotifMsg(hostId, ngamsCfgObj, msgId, retentionBuf_)
else:
# Emission of the Notification Message is forced. Just go ahead
# and send it out.
_sendNotifMsg(hostId, ngamsCfgObj, type, subject, dataRef, recList,
contentType, attachmentName, dataInFile)
# Release Notification Semaphore.
notifSem_.release()
except Exception:
# Release Notification Semaphore.
notifSem_.release()
raise
def addDiskHistEntry(self,
hostId,
diskId,
synopsis,
descrMimeType = None,
descr = None,
origin = None,
date = None):
"""
Add an entry in the NGAS Disks History Table (ngas_disks_hist)
indicating a major action or event occurring in the context of a disk.
dbConObj: Instance of NG/AMS DB class (ngamsDbBase).
diskId: Disk ID for the disk concerned (string).
synopsis: A short description of the action
(string/max. 255 char.s).
descrMimeType: The mime-type of the contents of the description
field. Must be specified when a description is
given (string).
descr: An arbitrary long description of the action or event
in the life-time of the disk (string).
origin: Origin of the history log entry. Can either be the
name of an application or the name of an operator.
If not specified (= None) it will be set to
'NG/AMS - <host name>' (string).
date: Date for adding the log entry. If not specified (set
to None), the function takes the current date and
writes this in the new entry (string/ISO 8601).
Returns: Void.
"""
T = TRACE()
try:
if (origin == None):
origin = "NG/AMS@" + hostId
now = time.time()
if (date == None):
histDate = self.convertTimeStamp(now)
else:
histDate = self.convertTimeStamp(date)
if (descr != None):
if (descrMimeType == None):
errMsg = "Mime-type must be specified for entry in the "+\
"NGAS Disks History when a Description is given!"
raise Exception, errMsg
if (descrMimeType == NGAMS_XML_MT):
descr = re.sub("\n", "", descr)
descr = re.sub("> *<", "><", descr)
descr = re.sub(">\t*<", "><", descr)
descr = descr
else:
descr = "None"
if (descrMimeType != None):
mt = descrMimeType
else:
mt = "None"
sqlQuery = ("INSERT INTO ngas_disks_hist "
"(disk_id, hist_date, hist_origin, hist_synopsis, "
"hist_descr_mime_type, hist_descr) VALUES "
"({0}, {1}, {2}, {3}, {4}, {5})")
self.query2(sqlQuery, args = (diskId, histDate, origin, synopsis, mt, descr))
logger.info("Added entry in NGAS Disks History Table - Disk ID: %s - Date: %s - " + \
"Origin: %s - Synopsis: %s - Description Mime-type: %s - Description: %s",
diskId, toiso8601(now, local=True), origin, synopsis, str(mt), str(descr))
self.triggerEvents()
except Exception, e:
raise e
