def genSqlResult(self): """ Generate a list with the information as read from ngas_files and defined by the variables ngamsDbCore.NGAS_FILES_DISK_ID, ... . Returns: List with information from one row of ngas_files (list). """ ing_date = '' creation_date = '' if self.__ingestionDate is not None: ing_date = toiso8601(self.__ingestionDate, local=True) if self.__creationDate is not None: creation_date = toiso8601(self.__creationDate, local=True) return [ self.getDiskId(), self.getFilename(), self.getFileId(), int(self.getFileVersion()), self.getFormat(), int(self.getFileSize()), int(self.getUncompressedFileSize()), self.getCompression(), ing_date, int(self.getIgnore()), self.getChecksum(), self.getChecksumPlugIn(), self.getFileStatus(), creation_date, self.getIoTime(), self.getIngestionRate(), self.getContainerId() ]
def getObjStatus(self): """ Return a list with the current status of the object. The format of the list is: [[<xml attribute name>, <value>, ...], ...] Returns: List with object status (list/list). """ compDate = "" if self.__completionDate is None else toiso8601( self.__completionDate) instDate = "" if self.__installationDate is None else toiso8601( self.__installationDate) lastCheck = "" if self.__lastCheck is None else toiso8601( self.__lastCheck) return [["DiskId", self.getDiskId()], ["Archive", self.getArchive()], ["InstallationDate", instDate], ["Type", self.getType()], ["Manufacturer", self.getManufacturer()], ["LogicalName", self.getLogicalName()], ["HostId", self.getHostId()], ["SlotId", self.getSlotId()], ["Mounted", self.getMounted()], ["MountPoint", self.getMountPoint()], ["NumberOfFiles", self.getNumberOfFiles()], ["AvailableMb", self.getAvailableMb()], ["BytesStored", self.getBytesStoredStr()], ["Completed", self.getCompleted()], ["CompletionDate", compDate], ["Checksum", self.getChecksum()], ["TotalDiskWriteTime", self.getTotalDiskWriteTime()], ["LastCheck", lastCheck]]
def prepNgasDiskInfoFile(hostId, diskInfoObj, yankNewlines=0, yankElSpaces=0): """ Create and prepare an NGAS Disk Info Status Object and set the attributes in the ngamsStatus object accordingly. The NgasDiskInfo XML document is returned. diskInfoObj: NGAS Disk Info Object containing information about the disk for which to generate the NgasDiskInfo file (ngasDiskInfo). yankNewlines: If set to 1 newline characters will be removed from the resulting XML document (integer/0|1). yankElSpaces: Yank spaces between elements (integer/0|1). Returns: NgasDiskInfo XML document (string/xml). """ status = ngamsStatus.ngamsStatus() status.\ setDate(toiso8601()).\ setVersion(getNgamsVersion()).setHostId(hostId).\ setMessage("Disk status file").addDiskStatus(diskInfoObj) xmlDoc = status.genXml(0, 1, 0, 0, 1).toprettyxml() if (yankNewlines): xmlDoc = re.sub("\n", "", str(xmlDoc)).strip() if (yankElSpaces): xmlDoc = re.sub("> *<", "><", xmlDoc) xmlDoc = re.sub(">\t*<", "><", xmlDoc) return xmlDoc
def convertTimeStamp(self, t): """ Convert a timestamp given in one of the following formats: """ if isinstance(t, basestring): return t else: return toiso8601(t, local=True)
def getObjStatus(self): """ Return a list with the current status of the object. The format of the list is: [[<xml attribute name>, <value>, ...], ...] Returns: List with object status (list/list). """ # Fields in XML document/ASCII dump ing_date = '' creation_date = '' mod_date = '' acc_date = '' if self.__ingestionDate is not None: ing_date = toiso8601(self.__ingestionDate) if self.__creationDate is not None: creation_date = toiso8601(self.__creationDate) if self.__modDate is not None: mod_date = toiso8601(self.__modDate) if self.__accDate is not None: acc_date = toiso8601(self.__accDate) return [["DiskId", self.getDiskId()], ["FileName", self.getFilename()], ["FileId", self.getFileId()], ["FileVersion", self.getFileVersion()], ["Format", self.getFormat()], ["FileSize", self.getFileSize()], ["UncompressedFileSize", self.getUncompressedFileSize()], ["Compression", self.getCompression()], ["IngestionDate", ing_date], ["Ignore", self.getIgnore()], ["Checksum", self.getChecksum()], ["ChecksumPlugIn", self.getChecksumPlugIn()], ["FileStatus", self.getFileStatus()], ["CreationDate", creation_date], ["Tag", self.getTag()], ["Permissions", self.getPermissions()], ["Owner", self.getOwner()], ["Group", self.getGroup()], ["ModificationDate", mod_date], ["AccessDate", acc_date], ["TotalIoTime", self.getIoTime()], ["IngestionRate", self.getIngestionRate()], ["ContainerId", self.getContainerId()]]
def _dummy_stat(host_id, status, msg, data=None): stat = ngamsStatus().\ setDate(toiso8601()).\ setVersion(getNgamsVersion()).setHostId(host_id).\ setStatus(status).\ setMessage(msg).\ setState(NGAMS_ONLINE_STATE).\ setSubState(NGAMS_IDLE_SUBSTATE) if data: stat.setData(data) return stat
def getObjStatus(self): """ Return a list with the current status of the object. The format of the list is: [[<xml attribute name>, <value>, ...], ...] Returns: List with object status (list/list). """ # Fields in XML document/ASCII dump wakeup_time = "" inst_date = "" if self.__srvReqWakeUpTime is not None: wakeup_time = toiso8601(self.__srvReqWakeUpTime) if self.__installationDate is not None: inst_date = toiso8601(self.__installationDate) return [["Host ID", self.getHostId()], ["Domain", self.getDomain()], ["IP Address", self.getIpAddress()], ["MAC Address", self.getMacAddress()], ["Number of Slots", self.getNSlots()], ["Cluster Name", self.getClusterName()], ["Installation Date", inst_date], ["Server Version", self.getSrvVersion()], ["Server Port", self.getSrvPort()], ["Server Allow Archiving", self.getSrvArchive()], ["Server Allow Retrieving", self.getSrvRetrieve()], ["Server Allow Processing", self.getSrvProcess()], ["Server Allow Remove", self.getSrvRemove()], ["Server Data Checking", self.getSrvDataChecking()], ["Server State", self.getSrvState()], ["Server Suspended", self.getSrvSuspended()], ["Server Wake-Up Request", self.getSrvReqWakeUpSrv()], ["Server Wake-Up Time", wakeup_time]]
def dumpBuf(self): """ Dump contents of object into a string buffer. Returns: String buffer with disk info (string). """ format = prFormat1() buf = "Subscriber Info:\n" buf += format % ("Host ID:", self.getHostId()) if (self.getPortNo() > 0): buf += format % ("Port Number:", str(self.getPortNo())) buf += format % ("Priority:", str(self.getPriority())) buf += format % ("Subscriber ID:", self.getId()) buf += format % ("Subscriber URL:", self.getUrl()) if self.__startDate is not None: buf += format % ("Start Date:", toiso8601(self.__startDate)) buf += format % ("Filter Plug-In:", self.getFilterPi()) buf += format % ("Filter Plug-In Par.s:", self.getFilterPiPars()) if self.__lastFileIngDate is not None: buf += format % ("Last File Ing. Date:", toiso8601(self.__lastFileIngDate)) return buf
def dump(self): """ Create an ASCII dump of the contents in a string buffer and return a reference to the buffer. Returns: String buffer containing the ASCII dump (string). """ T = TRACE() buf = "Contents of ngamsMirroringRequest:%s:\n" % str(self) buf += "Instance ID: %s\n" % self.getInstanceId() buf += "File ID: %s\n" % self.getFileId() buf += "File Version: %d\n" % self.getFileVersion() if self.__ingestionDate is not None: buf += "Ingestion Date: %s\n" % toiso8601( self.__ingestionDate) buf += "Server List ID: %d\n" % self.getSrvListId() buf += "XML File Info:\n%s\n" % self.getXmlFileInfo() buf += "Status: %s\n" % self.getStatusAsStr() buf += "Message: %s\n" % str(self.getMessage()) buf += "Last Activity Date: %s\n" % toiso8601( self.getLastActivityTime()) return buf
def _genXml(self, doc): contEl = doc.createElement('Container') contEl.setAttribute('id', str( self.getContainerId())) # might be an uuid.uuid4 object contEl.setAttribute('name', self.getContainerName()) contEl.setAttribute('size', str(self.getContainerSize())) if self._ingestionDate is not None: contEl.setAttribute('ingestionDate', toiso8601(self._ingestionDate)) for fileInfo in self._filesInfo: contEl.appendChild(fileInfo.genXml()) for childCont in self._containers: contEl.appendChild(childCont._genXml(doc)) return contEl
def genSummary(self): """ Generate a summary of the contents of the object in one line. Returns: Reference to string buffer with the object summary (string). """ T = TRACE(5) buf = "Summary of Mirroring Request Object: " buf += "Instance ID: %s" % self.getInstanceId() buf += ". File ID: %s" % self.getFileId() buf += ". File Version: %d" % self.getFileVersion() buf += ". Status: %s" % self.getStatusAsStr() buf += ". Message: %s" % str(self.getMessage()) buf += ". Last Activity Date: %s" % toiso8601( self.getLastActivityTime()) return buf
def genXml(self): """ Generate an XML DOM Node and with the information in the object and return this. Returns: DOM node object (Node). """ tmpSubscrEl = xml.dom.minidom.Document().createElement("Subscriber") tmpSubscrEl.setAttribute("HostId", self.getHostId()) if (self.getPortNo() > 0): tmpSubscrEl.setAttribute("PortNo", str(self.getPortNo())) tmpSubscrEl.setAttribute("Priority", str(self.getPriority())) tmpSubscrEl.setAttribute("SubscriberUrl", self.getUrl()) if self.__startDate is not None: tmpSubscrEl.setAttribute("StartDate", toiso8601(self.__startDate)) tmpSubscrEl.setAttribute("FilterPlugIn", self.getFilterPi()) tmpSubscrEl.setAttribute("FilterPlugInPars", self.getFilterPiPars()) return tmpSubscrEl
def createDbFileChangeStatusDoc(self, hostId, operation, fileInfoObjList, diskInfoObjList=[]): """ The function creates a pickle document in the '<Disk Mt Pt>/.db/cache' directory from the information in the 'fileInfoObj' object. operation: Has to be either ngams.NGAMS_DB_CH_FILE_INSERT or ngams.NGAMS_DB_CH_FILE_DELETE (string). fileInfoObj: List of instances of NG/AMS File Info Object containing the information about the file (list/ngamsFileInfo). diskInfoObjList: It is possible to give the information about the disk(s) in question via a list of ngamsDiskInfo objects (list/ngamsDiskInfo). Returns: Void. """ T = TRACE() # TODO: Implementation concern: This class is suppose to be # at a lower level in the hierarchie than the ngamsFileInfo, # ngamsFileList, ngamsDiskInfo and ngamsStatus classes and as # such these should not be used from within this class. All usage # of these classes in the ngamsDbBase class should be analyzed. # Probably these classes should be made base classes for this class. # TODO: Potential memory bottleneck. timeStamp = toiso8601() # Sort the File Info Objects according to disks. fileInfoObjDic = {} for fileInfo in fileInfoObjList: if (not fileInfoObjDic.has_key(fileInfo.getDiskId())): fileInfoObjDic[fileInfo.getDiskId()] = [] fileInfoObjDic[fileInfo.getDiskId()].append(fileInfo) # Get the mount points for the various disks concerned. mtPtDic = {} if (diskInfoObjList == []): diskIdMtPtList = self.getDiskIdsMtPtsMountedDisks(hostId) for diskId, mtPt in diskIdMtPtList: mtPtDic[diskId] = mtPt else: for diskInfoObj in diskInfoObjList: mtPtDic[diskInfoObj.getDiskId()] = diskInfoObj.getMountPoint() # Create on each disk the relevant DB Change Status Document. tmpFileList = None for diskId in fileInfoObjDic.keys(): if ((len(fileInfoObjList) == 1) and (diskInfoObjList == [])): tmpStatObj = fileInfoObjList[0].setTag(operation) else: # Apparently the dbId is used only as the ID of the file list, # which is totally irrelevant (this object seems to be read on # ngamsJanitorThread#checkDbChangeCache) dbId = diskId + "." + timeStamp tmpFileObjList = fileInfoObjDic[diskId] tmpFileList = ngamsFileList.ngamsFileList(dbId, operation) for fileObj in tmpFileObjList: tmpFileList.addFileInfoObj(fileObj) tmpStatObj = ngamsStatus.ngamsStatus().\ setDate(timeStamp).\ setVersion(getNgamsVersion()).\ setHostId(hostId).\ setMessage(dbId).\ addFileList(tmpFileList) two_stage_pickle(mtPtDic[diskId], tmpStatObj)
def genXml(self, genCfgStatus=0, genDiskStatus=0, genFileStatus=0, genStatesStatus=1, genLimDiskStatus=0): """ Generate an XML Node which contains the status report (root node). genCfgStatus: 1 = generate configuration status (integer). genDiskStatus: 1 = generate disk status (integer). genFileStatus: 1 = generate file status (integer). genStatesStatus: 1 = generate State/Sub-State (integer). genLimDiskStatus: 1 = generate only a limited (generic) set of disk status (integer). Returns: XML document (xml.dom.minidom.Document). """ T = TRACE(5) doc = xml.dom.minidom.Document() ngamsStatusEl = doc.createElement(NGAMS_XML_STATUS_ROOT_EL) doc.appendChild(ngamsStatusEl) # Status Element. statusEl = xml.dom.minidom.Document().createElement("Status") statusEl.setAttribute("Date", self.getDate()) statusEl.setAttribute("Version", self.getVersion()) statusEl.setAttribute("HostId", self.getHostId()) statusEl.setAttribute("Message", self.getMessage()) if (genStatesStatus): statusEl.setAttribute("Status", self.getStatus()) statusEl.setAttribute("State", self.getState()) statusEl.setAttribute("SubState", self.getSubState()) # Add the request handling status (if defined). if (self.getRequestId()): statusEl.setAttribute("RequestId", str(self.getRequestId())) if self.__requestTime is not None: statusEl.setAttribute("RequestTime", toiso8601(self.__requestTime)) if (self.getCompletionPercent()): statusEl.setAttribute("CompletionPercent", "%.2f" % self.getCompletionPercent()) if (self.getExpectedCount() != None): statusEl.setAttribute("ExpectedCount", str(self.getExpectedCount())) if (self.getActualCount() != None): statusEl.setAttribute("ActualCount", str(self.getActualCount())) if self.__estTotalTime is not None: statusEl.setAttribute("EstTotalTime", str(self.__estTotalTime)) if self.__remainingTime is not None: statusEl.setAttribute("RemainingTime", str(self.__remainingTime)) if self.__lastRequestStatUpdate is not None: statusEl.setAttribute("LastRequestStatUpdate", toiso8601(self.__lastRequestStatUpdate)) if self.__completionTime is not None: statusEl.setAttribute("CompletionTime", toiso8601(self.__completionTime)) ngamsStatusEl.appendChild(statusEl) # NgamsCfg Element. if (genCfgStatus): ngamsCfgEl = self.__ngamsCfg.genXml() ngamsStatusEl.appendChild(ngamsCfgEl) # DiskStatus Elements. if (genDiskStatus): for disk in self.getDiskStatusList(): diskStatusEl = disk.genXml(genLimDiskStatus, genFileStatus) ngamsStatusEl.appendChild(diskStatusEl) # FileList Elements. if (len(self.getFileListList())): for fileListObj in self.getFileListList(): fileListXmlNode = fileListObj.genXml() ngamsStatusEl.appendChild(fileListXmlNode) # Container Elements for container in self.getContainerList(): ngamsStatusEl.appendChild(container.genXml()) return doc
def notify(hostId, ngamsCfgObj, type, subject, dataRef, recList=[], force=0, contentType=None, attachmentName=None, dataInFile=0): """ Send an Notification Email to the subscriber(s) about an event happening. srvObj: Reference to NGAS server object (ngamsServer). type: Type of Notification (See NGAMS_NOTIF_* in ngams). subject: Subject of message (string). dataRef: Message to send or filename containing data (string). recList: List with recipients that should receive the Notification Message (list). force: Force distribution of Notification Message even though this was disabled in the configuration (integer/0|1). contentType: Mime-type of message (string). attachmentName: Name of attachment in mail (string). dataInFile: Indicates that data to be send is stored in a file (integer/0|1). Returns: Void. """ T = TRACE() if ((not force) or (ngamsCfgObj.getNotifActive() != 1)): logger.debug("Leaving notify() with no action (disabled/force=0)") return # Force emission if data is contained in a file and the (Notification # Service is activated). if (dataInFile): force = 1 global notifSem_ try: # Take Notification Semaphore. notifSem_.acquire() # Load a possible pickled Retention Buffer. pickleObjFile = _getNotifRetBufPickleFile(ngamsCfgObj, hostId) global retentionBuf_ if (not retentionBuf_): # Check if pickled Retention Object exists. if (os.path.exists(pickleObjFile)): retentionBuf_ = ngamsLib.loadObjPickleFile(pickleObjFile) else: retentionBuf_ = {} # The Retention Buffer is handled the following way: # # * If the Retention Buffer does not have an entry for that Message ID, # create a new entry for that Message ID + store the Last Emission # Time. # # * If the time since the Last Emission Time is less than the Max. # Retention Time from the configuration, we buffer this message # in the Retention Buffer. msgId = type + ":" + subject timeNow = time.time() maxRetTime = isoTime2Secs(ngamsCfgObj.getMaxRetentionTime()) maxRetNo = ngamsCfgObj.getMaxRetentionSize() if ((not force) and (not retentionBuf_.has_key(msgId))): # We simply create a new entry for this Message ID and initialize # the Last Emission Time. This message will be sent out. retentionBuf_[msgId] = [timeNow, None, type, subject, recList, []] ngamsLib.createObjPickleFile(pickleObjFile, retentionBuf_) _sendNotifMsg(hostId, ngamsCfgObj, type, subject, dataRef, recList, contentType, attachmentName) elif (not force): logger.debug("Appending Notification Message with ID: %s " + +\ "in Notification Retention Buffer", msgId) # Set the Retention Start Time if not set. if (not retentionBuf_[msgId][1]): retentionBuf_[msgId][1] = timeNow # Append the new element containing the information about the # message to be retained. isoTime = toiso8601(timeNow) retentionBuf_[msgId][5].append([isoTime, dataRef]) # Update the Notification Retention Buffer Pickle File and check # if there are messages to send out. ngamsLib.createObjPickleFile(pickleObjFile, retentionBuf_) _checkSendNotifMsg(hostId, ngamsCfgObj, msgId, retentionBuf_) else: # Emission of the Notification Message is forced. Just go ahead # and send it out. _sendNotifMsg(hostId, ngamsCfgObj, type, subject, dataRef, recList, contentType, attachmentName, dataInFile) # Release Notification Semaphore. notifSem_.release() except Exception: # Release Notification Semaphore. notifSem_.release() raise
def addDiskHistEntry(self, hostId, diskId, synopsis, descrMimeType = None, descr = None, origin = None, date = None): """ Add an entry in the NGAS Disks History Table (ngas_disks_hist) indicating a major action or event occurring in the context of a disk. dbConObj: Instance of NG/AMS DB class (ngamsDbBase). diskId: Disk ID for the disk concerned (string). synopsis: A short description of the action (string/max. 255 char.s). descrMimeType: The mime-type of the contents of the description field. Must be specified when a description is given (string). descr: An arbitrary long description of the action or event in the life-time of the disk (string). origin: Origin of the history log entry. Can either be the name of an application or the name of an operator. If not specified (= None) it will be set to 'NG/AMS - <host name>' (string). date: Date for adding the log entry. If not specified (set to None), the function takes the current date and writes this in the new entry (string/ISO 8601). Returns: Void. """ T = TRACE() try: if (origin == None): origin = "NG/AMS@" + hostId now = time.time() if (date == None): histDate = self.convertTimeStamp(now) else: histDate = self.convertTimeStamp(date) if (descr != None): if (descrMimeType == None): errMsg = "Mime-type must be specified for entry in the "+\ "NGAS Disks History when a Description is given!" raise Exception, errMsg if (descrMimeType == NGAMS_XML_MT): descr = re.sub("\n", "", descr) descr = re.sub("> *<", "><", descr) descr = re.sub(">\t*<", "><", descr) descr = descr else: descr = "None" if (descrMimeType != None): mt = descrMimeType else: mt = "None" sqlQuery = ("INSERT INTO ngas_disks_hist " "(disk_id, hist_date, hist_origin, hist_synopsis, " "hist_descr_mime_type, hist_descr) VALUES " "({0}, {1}, {2}, {3}, {4}, {5})") self.query2(sqlQuery, args = (diskId, histDate, origin, synopsis, mt, descr)) logger.info("Added entry in NGAS Disks History Table - Disk ID: %s - Date: %s - " + \ "Origin: %s - Synopsis: %s - Description Mime-type: %s - Description: %s", diskId, toiso8601(now, local=True), origin, synopsis, str(mt), str(descr)) self.triggerEvents() except Exception, e: raise e