def unpackSqlResult(self, sqlResult):
"""
Unpack the result from an SQL query, whereby the columns of one row in
the ngas_subscribers table is queried with ngamsDb.getSubscriberInfo()
(specific Subscriber specified).
sqlResult: List with elements from the ngas_subscribers table (list).
Returns: Reference to object itself.
"""
T = TRACE()
self.setHostId(sqlResult[0]).\
setPortNo(sqlResult[1]).\
setPriority(sqlResult[2]).\
setId(sqlResult[3]).\
setUrl(sqlResult[4]).\
setFilterPi(sqlResult[6]).\
setFilterPiPars(sqlResult[7]).\
setLastFileIngDate(fromiso8601(sqlResult[8], local=True)).\
setConcurrentThreads(sqlResult[9])
if sqlResult[5]:
self.setStartDate(fromiso8601(sqlResult[5], local=True))
else:
self.setStartDate(None)
return self
def unpackSqlResult(self, sqlQueryRes):
"""
Sets the members of the class from the query information as
returned from ngamsDb.getFileInfoFromFileIdHostId().
sqlQueryRes: Query information (list).
Return: Reference to object itself.
"""
T = TRACE(5)
self.setDiskId(sqlQueryRes[0])
self.setFilename(sqlQueryRes[1])
self.setFileId(sqlQueryRes[2])
self.setFileVersion(sqlQueryRes[3])
self.setFormat(sqlQueryRes[4])
self.setFileSize(sqlQueryRes[5])
uncomprSz = sqlQueryRes[6]
self.setUncompressedFileSize(uncomprSz)
self.setCompression(sqlQueryRes[7])
if sqlQueryRes[8]:
self.setIngestionDate(fromiso8601(sqlQueryRes[8], local=True))
self.setIgnore(sqlQueryRes[9])
self.setChecksum(sqlQueryRes[10])
self.setChecksumPlugIn(sqlQueryRes[11])
self.setFileStatus(sqlQueryRes[12])
if sqlQueryRes[13]:
self.setCreationDate(fromiso8601(sqlQueryRes[13], local=True))
self.setIoTime(sqlQueryRes[14])
self.setIngestionRate(sqlQueryRes[15])
self.setContainerId(sqlQueryRes[16])
return self
def unpackFromDomNode(self, diskNode, ignoreVarDiskPars=0):
"""
Unpack the disk information contained in a DOM DiskStatus
Node and set the members of the object accordingly.
diskNode: DOM Disk Node (Node).
ignoreVarDiskPars: Ignore the variable part of the disk status:
Host ID, Slot ID, Mounted, Mount
Point (integer/0|1).
Returns: Reference to object itself.
"""
T = TRACE()
self.setDiskId(getAttribValue(diskNode, "DiskId"))
self.setArchive(getAttribValue(diskNode, "Archive"))
instDate = getAttribValue(diskNode, "InstallationDate")
if instDate:
self.setInstallationDate(fromiso8601(instDate))
self.setType(getAttribValue(diskNode, "Type"))
self.setManufacturer(getAttribValue(diskNode, "Manufacturer", 1))
self.setLogicalName(getAttribValue(diskNode, "LogicalName"))
# These attributes are not contained in certain Status XML
# Documents, e.g. in the NgasDiskInfo files.
if (not ignoreVarDiskPars):
self.setHostId(getAttribValue(diskNode, "HostId"))
self.setSlotId(getAttribValue(diskNode, "SlotId"))
self.setMounted(getAttribValue(diskNode, "Mounted"))
self.setMountPoint(getAttribValue(diskNode, "MountPoint"))
self.setNumberOfFiles(getAttribValue(diskNode, "NumberOfFiles"))
self.setAvailableMb(getAttribValue(diskNode, "AvailableMb"))
self.setBytesStored(getAttribValue(diskNode, "BytesStored"))
self.setCompleted(getAttribValue(diskNode, "Completed"))
compDate = getAttribValue(diskNode, "CompletionDate", 1)
if compDate:
self.setCompletionDate(fromiso8601(compDate))
self.setChecksum(getAttribValue(diskNode, "Checksum"))
write_time = getAttribValue(diskNode, "TotalDiskWriteTime")
if write_time:
self.setTotalDiskWriteTime(float(write_time))
# Handle files.
fileNodes = diskNode.getElementsByTagName("FileStatus")
for fileNode in fileNodes:
fileInfo = ngamsFileInfo.ngamsFileInfo().\
unpackFromDomNode(fileNode, self.getDiskId())
self.addFileObj(fileInfo)
return self
def unpackFromDomNode(self, fileNode, diskId=None):
"""
Unpack the file information contained in a DOM FileStatus
Node and set the members of the object accordingly.
fileNode: DOM File Node object (DOM object).
diskId: Disk ID for disk where file is stored (string).
Returns: Reference to object itself.
"""
if (diskId != None):
self.setDiskId(diskId)
else:
self.setDiskId(getAttribValue(fileNode, "DiskId", 1))
uncomprSize = getAttribValue(fileNode, "UncompressedFileSize", 1)
checksumPi = getAttribValue(fileNode, "ChecksumPlugIn", 1)
self.\
setChecksum(getAttribValue(fileNode, "Checksum", 1)).\
setChecksumPlugIn(checksumPi).\
setCompression(getAttribValue(fileNode, "Compression", 1)).\
setFileId(getAttribValue(fileNode, "FileId", 1)).\
setFilename(getAttribValue(fileNode, "FileName", 1)).\
setUncompressedFileSize(uncomprSize).\
setFileSize(getAttribValue(fileNode, "FileSize", 1)).\
setFileStatus(getAttribValue(fileNode, "FileStatus", 1)).\
setFileVersion(getAttribValue(fileNode, "FileVersion", 1)).\
setFormat(getAttribValue(fileNode, "Format", 1)).\
setGroup(getAttribValue(fileNode, "Group", 1)).\
setIgnore(getAttribValue(fileNode, "Ignore", 1)).\
setOwner(getAttribValue(fileNode, "Owner", 1)).\
setPermissions(getAttribValue(fileNode, "Permissions", 1)).\
setTag(getAttribValue(fileNode, "Tag", 1)).\
setIoTime(getAttribValue(fileNode, "TotalIoTime", 1)).\
setIngestionRate(getAttribValue(fileNode, "IngestionRate", 1)).\
setContainerId(getAttribValue(fileNode, "ContainerId", 1))
creation_date = getAttribValue(fileNode, "CreationDate", 1)
if creation_date:
self.setCreationDate(fromiso8601(creation_date))
ingestion_date = getAttribValue(fileNode, "IngestionDate", 1)
if ingestion_date:
self.setIngestionDate(fromiso8601(ingestion_date))
mod_date = getAttribValue(fileNode, "ModificationDate", 1)
if mod_date:
self.setModDate(fromiso8601(mod_date))
acc_date = getAttribValue(fileNode, "ModificationDate", 1)
if acc_date:
self.setAccDate(fromiso8601(acc_date))
return self
def getLastDiskCheck(self, hostId=""):
"""
Queries all the Last Check Flags for all disks or for all disks
currently mounted in a specific host. A Dictionary is returned
containining the Disk IDs as keys, and the time for the last check
in seconds. If the value is NULL in the DB, it is set to 0.
hostId: If specified, only disks mounted in this system are
taken into account (string).
Returns: Dictionary with entry for each disk. Key is Disk
ID (dictionary).
"""
sql = []
vals = []
sql.append("SELECT disk_id, last_check from ngas_disks")
if hostId:
sql.append(" WHERE host_id = {}")
vals.append(hostId)
res = self.query2(''.join(sql), args=vals)
diskDic = {}
for diskInfo in res:
if (not diskInfo[1]):
timeSinceLastCheck = 0
else:
timeSinceLastCheck = fromiso8601(diskInfo[1], local=True)
diskDic[diskInfo[0]] = timeSinceLastCheck
return diskDic
def unpackFromDomNode(self, contEl):
"""
Unpacks the contents of the given DOM Element
and populates this ngamsContainer object with
them. This method also recursively traverses
any children elements containing information
about child containers and files.
:param contEl: xml.dom.minidom.Element
"""
self.setContainerId(contEl.getAttribute('id'))
self.setContainerName(contEl.getAttribute('name'))
self.setContainerSize(int(contEl.getAttribute('size')))
ingDate = contEl.getAttribute('ingestionDate')
self.setIngestionDate(fromiso8601(ingDate) if ingDate else None)
subContEls = [n for n in contEl.childNodes \
if n.nodeType == minidom.Node.ELEMENT_NODE
and n.tagName == 'Container']
fileEls = [n for n in contEl.childNodes \
if n.nodeType == minidom.Node.ELEMENT_NODE
and n.tagName == 'FileStatus']
for subContEl in subContEls:
childContainer = ngamsContainer()
childContainer.unpackFromDomNode(subContEl)
self.addContainer(childContainer)
for fileEl in fileEls:
fileInfo = ngamsFileInfo.ngamsFileInfo()
fileInfo.unpackFromDomNode(fileEl)
self.addFileInfo(fileInfo)
def fromTimestamp(self, timestamp):
"""
Converts a database timestamp into a number of seconds from the epoch.
This is the reverse of `asTimestamp`.
"""
if timestamp is None:
return None
return fromiso8601(timestamp, local=True)
def getSubscriberStatus(self,
subscrIds,
hostId = "",
portNo = -1):
"""
Method to query the information about the Ingestion Date of the
last file delivered to the Subscriber. A list is returned, which
contains the following:
[(<Subscriber ID>, <Last File Ingestion Date (ISO 8601)>), ...]
subscrIds: List of Subscriber ID to query (list/string).
hostId: Host name of Subscriber host (string).
portNo: Port number used by Subscriber host (integer).
Returns: List with Subscriber status (list/tuple/string).
"""
T = TRACE()
if not subscrIds:
return []
sql = []
vals = []
sql_tmp = ("SELECT subscr_id, last_file_ingestion_date "
"FROM ngas_subscribers WHERE subscr_id IN (%s)")
params = []
for i in subscrIds:
params.append('{}')
vals.append(i)
sql_tmp = sql_tmp % ','.join(params)
sql.append(sql_tmp)
if hostId:
sql.append(" AND host_id = {}")
vals.append(hostId)
if portNo != -1:
sql.append(" AND srv_port = {}")
vals.append(portNo)
res = self.query2(''.join(sql), args = vals)
if not res:
return []
subscrStatus = []
for subscrInfo in res:
if subscrInfo[1]:
lastIngDate = fromiso8601(subscrInfo[1], local=True)
else:
lastIngDate = None
subscrStatus.append((subscrInfo[0], lastIngDate))
return subscrStatus
def unpackSqlResult(self, sqlResult):
"""
Unpack the result from an SQL query, whereby the columns
of the ngas_disks table were queried as described in documentation
for ngamsDb.getDiskInfoFromDiskId().
sqlResult: Result of the SQL query (tuple).
Returns: Reference to object itself.
"""
T = TRACE(5)
self.\
setDiskId(sqlResult[0]).\
setArchive(sqlResult[1]).\
setLogicalName(sqlResult[2]).\
setHostId(sqlResult[3]).\
setSlotId(sqlResult[4]).\
setMounted(sqlResult[5]).\
setMountPoint(sqlResult[6]).\
setNumberOfFiles(sqlResult[7]).\
setAvailableMb(sqlResult[8]).\
setBytesStored(sqlResult[9]).\
setType(sqlResult[10]).\
setManufacturer(sqlResult[12]).\
setChecksum(sqlResult[14]).\
setTotalDiskWriteTime(sqlResult[15]).\
setCompleted(sqlResult[16])
if sqlResult[13]:
self.setInstallationDate(fromiso8601(sqlResult[13], local=True))
if sqlResult[17]:
self.setCompletionDate(fromiso8601(sqlResult[17], local=True))
if sqlResult[18]:
self.setLastCheck(fromiso8601(sqlResult[18], local=True))
if sqlResult[19]: self.setLastHostId(sqlResult[19])
return self
def getMinLastDiskCheck(self, hostId):
"""
Get the timestamp for the disk that was checked longest time ago
for all disks mounted in a specific NGAS Host.
hostId: Host ID of the host to consider (string).
Returns: Time since the 'oldest', last check (seconds since epoch)
(integer).
"""
T = TRACE()
sql = "SELECT min(last_check) FROM ngas_disks WHERE host_id={}"
res = self.query2(sql, args=(hostId, ))
if not res:
return None
val = res[0][0]
return fromiso8601(val, local=True)
def getIngDate(self, diskId, fileId, fileVersion):
"""
Get the ingestion date for the file.
diskId: ID of disk hosting the file (string).
fileId: ID of file to be deleted. No wildcards accepted
(string).
fileVersion: Version of file to delete (integer)
Returns: Ingestion date for file or None if file not found
(string/ISO 8601 | None).
"""
T = TRACE()
sql = "SELECT ingestion_date FROM ngas_files WHERE disk_id={0} AND file_id={1} AND file_version={2}"
res = self.query2(sql, args=(diskId, fileId, fileVersion))
if res:
if (res[0][0] == None):
return None
return fromiso8601(res[0][0], local=True)
return None
def unpackFromSqlQuery(self, sqlQueryResult):
"""
Unpack the host information from an SQL query result and set the
members of the class.
sqlQueryResult: SQL query result (list).
Returns: Reference to object itself.
"""
res = sqlQueryResult
# Static host information.
self.setHostId(res[0])
self.setDomain(res[1])
self.setIpAddress(res[2])
self.setMacAddress(res[3])
self.setNSlots(res[4])
self.setClusterName(res[5])
if res[6]:
self.setInstallationDate(fromiso8601(res[6], local=True))
# Host information set by server.
self.setSrvVersion(res[7])
self.setSrvPort(res[8])
self.setSrvArchive(res[9])
self.setSrvRetrieve(res[10])
self.setSrvProcess(res[11])
self.setSrvRemove(res[12])
self.setSrvDataChecking(res[13])
self.setSrvState(res[14])
# Parameters in connection with suspension
self.setSrvSuspended(res[15])
self.setSrvReqWakeUpSrv(res[16])
self.setSrvReqWakeUpTime(res[17])
return self
def unpackXmlDoc(self, doc, getStatus=0, ignoreVarDiskPars=0):
"""
Unpack a status report stored in an XML document and set the
members of the class accordingly.
doc: Status report as XML document (string).
getStatus: Extract also the status information from the
XML document (0|1/integer).
ignoreVarDiskPars: Ignore the variable part of the disk status:
Host ID, Slot ID, Mounted, Mount
Point (integer/0|1).
Returns: Reference to object itself.
"""
dom = xml.dom.minidom.parseString(doc)
ngamsStatusEl = ngamsGetChildNodes(dom, NGAMS_XML_STATUS_ROOT_EL)[0]
# Get the information from the Status Element.
nodeList = dom.getElementsByTagName("Status")
self.setDate(getAttribValue(nodeList[0], "Date"))
self.setVersion(getAttribValue(nodeList[0], "Version"))
self.setHostId(getAttribValue(nodeList[0], "HostId"))
if (getStatus):
self.setStatus(getAttribValue(nodeList[0], "Status"))
self.setMessage(getAttribValue(nodeList[0], "Message"))
if (getStatus):
self.setState(getAttribValue(nodeList[0], "State"))
self.setSubState(getAttribValue(nodeList[0], "SubState"))
# Get the optional request handling status.
requestId = getAttribValue(nodeList[0], "RequestId", 1)
if (requestId): self.setRequestId(requestId)
requestTime = getAttribValue(nodeList[0], "RequestTime", 1)
if (requestTime): self.setRequestTime(fromiso8601(requestTime))
completionPercent = getAttribValue(nodeList[0], "CompletionPercent", 1)
if (completionPercent): self.setCompletionPercent(completionPercent)
expectedCount = getAttribValue(nodeList[0], "ExpectedCount", 1)
if (expectedCount): self.setExpectedCount(expectedCount)
actualCount = getAttribValue(nodeList[0], "ActualCount", 1)
if (actualCount): self.setActualCount(actualCount)
estTotalTime = getAttribValue(nodeList[0], "EstTotalTime", 1)
if (estTotalTime): self.setEstTotalTime(float(estTotalTime))
remainingTime = getAttribValue(nodeList[0], "RemainingTime", 1)
if (remainingTime): self.setRemainingTime(float(remainingTime))
lastRequestStatUpdate = getAttribValue(nodeList[0],
"LastRequestStatUpdate", 1)
if (lastRequestStatUpdate):
self.setLastRequestStatUpdate(fromiso8601(lastRequestStatUpdate))
completionTime = getAttribValue(nodeList[0], "CompletionTime", 1)
if (completionTime):
self.setCompletionTime(fromiso8601(completionTime))
# Unpack the NG/AMS Configuration information.
ngamsCfgRootNode = dom.getElementsByTagName("NgamsCfg")
if (len(ngamsCfgRootNode) > 0):
tolerant = 1
self.__ngamsCfg.unpackFromRootNode(ngamsCfgRootNode[0], tolerant)
# Unpack Disk Status Elements and File Status Elements.
diskNodes = dom.getElementsByTagName("DiskStatus")
for diskNode in diskNodes:
diskInfo = ngamsDiskInfo.ngamsDiskInfo().\
unpackFromDomNode(diskNode, ignoreVarDiskPars)
self.addDiskStatus(diskInfo)
# Unpack File Lists.
fileListNodes = dom.getElementsByTagName("FileList")
for fileListNode in fileListNodes:
fileListObj = ngamsFileList.ngamsFileList()
self.addFileList(fileListObj)
fileListObj.unpackFromDomNode(fileListNode)
# Unpack Containers. Loop over the root containers,
# then let them deal with their own recursion
rootContainerEls = [node for node in ngamsStatusEl.childNodes \
if node.nodeType == xml.dom.minidom.Node.ELEMENT_NODE\
and node.tagName == 'Container']
for rootContainerEl in rootContainerEls:
container = ngamsContainer.ngamsContainer()
container.unpackFromDomNode(rootContainerEl)
self.addContainer(container)
dom.unlink()
return self
