def _checkDbInfo(testSuiteObj, testName, dbConObj, diskId):
"""
Dump the information in ngas_disks, ngas_files and ngas_files_retired in
connection with the disk and compare the results to reference dumps.
testSuiteObj: Test suite object (ngamsTestSuite)
testName: Name of the test case (string).
dbConObj: DB connection object (ngamsDb).
diskId: ID of disk to retire (string).
Returns: Void.
"""
# ngas_disks:
tmpDiskInfo = ngamsDiskInfo.ngamsDiskInfo().read(dbConObj, diskId)
diskInfoBuf = filterOutLines(
tmpDiskInfo.dumpBuf(),
["InstallationDate:", "AvailableMb:", "TotalDiskWriteTime:"])
refStatFile = "ref/ngasRetireDisk_%s_1" % testName
tmpStatFile = saveInFile(None, diskInfoBuf)
testSuiteObj.checkFilesEq(refStatFile, tmpStatFile,
"Incorrect/missing info in ngas_disks")
# ngas_files:
query = "SELECT %s FROM ngas_files nf WHERE disk_id='%s'" %\
(ngamsDb._ngasFilesCols, diskId)
curObj = dbConObj.dbCursor(query)
fileInfoBuf = ""
while (1):
res = curObj.fetch(1)
if (res == []): break
tmpFileInfo = ngamsFileInfo.ngamsFileInfo().unpackSqlResult(res[0])
fileInfoBuf += filterOutLines(
tmpFileInfo.dumpBuf(), ["IngestionDate:", "CreationDate:"]) + "\n"
refStatFile = "ref/ngasRetireDisk_%s_2" % testName
tmpStatFile = saveInFile(None, fileInfoBuf)
testSuiteObj.checkFilesEq(refStatFile, tmpStatFile,
"Incorrect/missing info in ngas_files")
# ngas_files_retired:
query = "SELECT %s FROM ngas_files_retired nf WHERE disk_id='%s'" %\
(ngamsDb._ngasFilesCols, diskId)
curObj = dbConObj.dbCursor(query)
fileInfoBuf = ""
while (1):
res = curObj.fetch(1)
if (res == []): break
tmpFileInfo = ngamsFileInfo.ngamsFileInfo().unpackSqlResult(res[0])
fileInfoBuf += filterOutLines(
tmpFileInfo.dumpBuf(), ["IngestionDate:", "CreationDate:"]) + "\n"
refStatFile = "ref/ngasRetireDisk_%s_3" % testName
tmpStatFile = saveInFile(None, fileInfoBuf)
testSuiteObj.checkFilesEq(refStatFile, tmpStatFile,
"Incorrect/missing info in ngas_files")
def getFileInfoFromDiskIdFilename(self, diskId, filename):
"""
The method queries the file information for a file referred to by the
Disk ID for the disk hosting it and the filename as stored in the
NGAS DB.
diskId: ID for disk hosting the file (string).
filename: NGAS (relative) filename (string).
Returns: Return ngamsFileInfo object with the information for the
file if found or None if the file was not found
(ngamsFileInfo|None).
"""
T = TRACE()
# Query for the file.
sql = "SELECT %s FROM ngas_files nf WHERE nf.disk_id={0} AND nf.file_name={1}"
sql = sql % (ngamsDbCore.getNgasFilesCols(
self._file_ignore_columnname), )
# Execute the query directly and return the result.
res = self.query2(sql, args=(diskId, filename))
if res:
return ngamsFileInfo.ngamsFileInfo().unpackSqlResult(res[0])
else:
return None
def readHierarchy(self, containerId, includeFiles=False):
"""
Reads an ngamsContainer object from the database
and recursively populates it with its children containers.
:param containerId: string
:return ngamsContainer.ngamsContainer
"""
container = self.read(containerId)
if includeFiles:
# Always get the latest version of the files
# We do this on the software side to avoid any complex SQL query
# that might not work in some engines
sql = "SELECT %s FROM ngas_files nf WHERE container_id = {0} ORDER BY nf.file_id, nf.file_version DESC"
sql = sql % (ngamsDbCore.getNgasFilesCols(self._file_ignore_columnname),)
res = self.query2(sql, args=(containerId,))
prevFileId = None
for r in res:
fileId = r[ngamsDbCore.NGAS_FILES_FILE_ID]
if fileId == prevFileId:
continue
prevFileId = fileId
fileInfo = ngamsFileInfo.ngamsFileInfo().unpackSqlResult(r)
container.addFileInfo(fileInfo)
# Check if it contains children
res = self.query2("SELECT container_id FROM ngas_containers WHERE parent_container_id = {0}", args=(containerId,))
for r in res:
container.addContainer(self.readHierarchy(r[0], includeFiles))
return container
def unpackFromDomNode(self,
fileListNode):
"""
Unpack a File List from a DOM File List Node object.
fileListNode: DOM File List Node object (Node).
Returns: Reference to object itself.
"""
self.setId(getAttribValue(fileListNode, "Id"))
self.setComment(getAttribValue(fileListNode, "Comment", 1))
self.setStatus(getAttribValue(fileListNode, "Status", 1))
# Unpack the File Status Elements.
fileStatusNodes = ngamsGetChildNodes(fileListNode, "FileStatus")
for fileStatusNode in fileStatusNodes:
fileStatusObj = ngamsFileInfo.ngamsFileInfo()
self.addFileInfoObj(fileStatusObj)
fileStatusObj.unpackFromDomNode(fileStatusNode)
# Unpack File List Elements.
fileListNodes = ngamsGetChildNodes(fileListNode, "FileList")
for fileListNode in fileListNodes:
fileListObj = ngamsFileList()
self.addFileListObj(fileListObj)
fileListObj.unpackFromDomNode(fileListNode)
return self
def unpackFromDomNode(self, contEl):
"""
Unpacks the contents of the given DOM Element
and populates this ngamsContainer object with
them. This method also recursively traverses
any children elements containing information
about child containers and files.
:param contEl: xml.dom.minidom.Element
"""
self.setContainerId(contEl.getAttribute('id'))
self.setContainerName(contEl.getAttribute('name'))
self.setContainerSize(int(contEl.getAttribute('size')))
ingDate = contEl.getAttribute('ingestionDate')
self.setIngestionDate(fromiso8601(ingDate) if ingDate else None)
subContEls = [n for n in contEl.childNodes \
if n.nodeType == minidom.Node.ELEMENT_NODE
and n.tagName == 'Container']
fileEls = [n for n in contEl.childNodes \
if n.nodeType == minidom.Node.ELEMENT_NODE
and n.tagName == 'FileStatus']
for subContEl in subContEls:
childContainer = ngamsContainer()
childContainer.unpackFromDomNode(subContEl)
self.addContainer(childContainer)
for fileEl in fileEls:
fileInfo = ngamsFileInfo.ngamsFileInfo()
fileInfo.unpackFromDomNode(fileEl)
self.addFileInfo(fileInfo)
def unpackFromDomNode(self, diskNode, ignoreVarDiskPars=0):
"""
Unpack the disk information contained in a DOM DiskStatus
Node and set the members of the object accordingly.
diskNode: DOM Disk Node (Node).
ignoreVarDiskPars: Ignore the variable part of the disk status:
Host ID, Slot ID, Mounted, Mount
Point (integer/0|1).
Returns: Reference to object itself.
"""
T = TRACE()
self.setDiskId(getAttribValue(diskNode, "DiskId"))
self.setArchive(getAttribValue(diskNode, "Archive"))
instDate = getAttribValue(diskNode, "InstallationDate")
if instDate:
self.setInstallationDate(fromiso8601(instDate))
self.setType(getAttribValue(diskNode, "Type"))
self.setManufacturer(getAttribValue(diskNode, "Manufacturer", 1))
self.setLogicalName(getAttribValue(diskNode, "LogicalName"))
# These attributes are not contained in certain Status XML
# Documents, e.g. in the NgasDiskInfo files.
if (not ignoreVarDiskPars):
self.setHostId(getAttribValue(diskNode, "HostId"))
self.setSlotId(getAttribValue(diskNode, "SlotId"))
self.setMounted(getAttribValue(diskNode, "Mounted"))
self.setMountPoint(getAttribValue(diskNode, "MountPoint"))
self.setNumberOfFiles(getAttribValue(diskNode, "NumberOfFiles"))
self.setAvailableMb(getAttribValue(diskNode, "AvailableMb"))
self.setBytesStored(getAttribValue(diskNode, "BytesStored"))
self.setCompleted(getAttribValue(diskNode, "Completed"))
compDate = getAttribValue(diskNode, "CompletionDate", 1)
if compDate:
self.setCompletionDate(fromiso8601(compDate))
self.setChecksum(getAttribValue(diskNode, "Checksum"))
write_time = getAttribValue(diskNode, "TotalDiskWriteTime")
if write_time:
self.setTotalDiskWriteTime(float(write_time))
# Handle files.
fileNodes = diskNode.getElementsByTagName("FileStatus")
for fileNode in fileNodes:
fileInfo = ngamsFileInfo.ngamsFileInfo().\
unpackFromDomNode(fileNode, self.getDiskId())
self.addFileObj(fileInfo)
return self
def cloneCheckFile(diskId, fileId, fileVersion, dbConObj, pClientObj,
checkRep):
"""
Clone a file a check afterwards if it was successfully cloned.
diskId: ID of disk cloned (string).
fileId: ID of file cloned (string).
fileVersion: Version of file to check (integer).
dbConObj: DB Connection Object (ngamsDb).
pClientObj: Initiated instance of NG/AMS P-Client Object (ngamsPClient).
checkRep: Check report (string).
Returns: Updated Check Report (string).
"""
msg = "\n-> Attempting to clone file: %s/%s/%s" %\
(diskId, fileId, fileVersion)
print msg
msg += " - status: "
res = pClientObj.clone(fileId, diskId, fileVersion, wait=1)
if (res.getStatus() == "FAILURE"):
status = "FAILURE: " + str(res.getMessage()) + "\n"
else:
# Check if file was really cloned.
res = dbConObj.getFileInfoFromFileIdHostId(_getTargHost(), fileId,
fileVersion)
if (res == []):
status = "FAILURE: File not cloned!\n"
else:
fileInfo = ngamsFileInfo.ngamsFileInfo().\
unpackSqlResult(res)
tmpPars = [["disk_id", fileInfo.getDiskId()], ["file_id", fileId],
["file_version", fileVersion]]
res = pClientObj.sendCmdGen(getHostName(),
pClientObj.getPort(),
NGAMS_CHECKFILE_CMD,
wait=1,
pars=tmpPars)
status = res.getMessage() + "\n"
return msg + status
def setFileChecksum(self, hostId, fileId, fileVersion, diskId, checksum,
checksumPlugIn):
"""
Set the checksum value in the ngas_files table.
hostId: ID of this NGAS host
fileId: ID of file (string).
fileVersion: Version of file (integer).
diskId: ID of disk where file is stored (string).
checksum: Checksum of file (string).
checksumPlugIn: Name of plug-in used to generate the
checksum (string).
Returns: Reference to object itself.
"""
T = TRACE()
sql = ("UPDATE ngas_files SET checksum={}, checksum_plugin={}"
" WHERE file_id={} AND file_version={} AND disk_id={}")
vals = (checksum, checksumPlugIn, fileId, fileVersion, diskId)
self.query2(sql, args=vals)
# Create a File Removal Status Document.
if (self.getCreateDbSnapshot()):
dbFileInfo = self.getFileInfoFromFileIdHostId(
hostId, fileId, fileVersion, diskId)
tmpFileObj = ngamsFileInfo.ngamsFileInfo().\
unpackSqlResult(dbFileInfo)
self.createDbFileChangeStatusDoc(hostId, NGAMS_DB_CH_FILE_UPDATE,
[tmpFileObj])
self.triggerEvents()
return self
def _scanDir(srcDir, recursive, fileExtList):
"""
Scan the given directory file for files. If diretories are found in the
directory, these are returned in a list.
srcDir: Directory to scan for matching files (string).
recursive: Traverse the directory tree recursively (integer/0|1).
fileExtList: List with extensions to take into account (list).
Returns: List with ngamsFileInfo objects for matching files (list).
"""
info(4, "Entering _scanDir() ...")
srcDir = os.path.expanduser(srcDir)
info(2, "Scanning directory: %s ..." % srcDir)
fileInfoList = []
fileList = glob.glob(os.path.normpath("%s/*" % srcDir))
for file in fileList:
file = file.strip()
if (os.path.isdir(file) and recursive):
fileInfoList += _scanDir(file, recursive, fileExtList)
else:
# Extension matching?
for ext in fileExtList:
if ((file.find("." + ext) + len("." + ext)) == len(file)):
# The extension matches. Get the info for the file.
statInfo = os.stat(file)
tmpFileObj = ngamsFileInfo.ngamsFileInfo().\
setFilename(file).\
setAccDateFromSecs(statInfo[7]).\
setModDateFromSecs(statInfo[8]).\
setCreationDate(statInfo[9]).\
setFileSize(statInfo[6])
fileInfoList.append(tmpFileObj)
break
info(4, "Leaving _scanDir()")
return fileInfoList
def deleteFileInfo(self,
hostId,
diskId,
fileId,
fileVersion,
genSnapshot=1):
"""
Delete one record for a certain file in the NGAS DB.
CAUTION: IF THE DB USER WITH WHICH THERE IS LOGGED IN HAS PERMISSION
TO EXECUTE DELETE STATEMENTS, THE INFORMATION ABOUT THE
FILE(S) IN THE NGAS DB WILL BE DELETED! THIS INFORMATION
CANNOT BE RECOVERED!!
diskId: ID of disk hosting the file (string).
fileId: ID of file to be deleted. No wildcards accepted
(string).
fileVersion: Version of file to delete (integer)
genSnapshot: Generate Db Snapshot (integer/0|1).
Returns: Reference to object itself.
"""
T = TRACE()
# We have to update some fields of the disk hosting the file
# when we delete a file (number_of_files, available_mb,
# bytes_stored, also maybe later: checksum.
dbDiskInfo = self.getDiskInfoFromDiskId(diskId)
# getFileInfoList returns a generator, so we iterate once to get the first element
try:
dbFileInfo = next(self.getFileInfoList(diskId, fileId,
fileVersion))
except StopIteration:
msg = "Cannot remove file. File ID: %s, " +\
"File Version: %d, Disk ID: %s"
errMsg = msg % (fileId, fileVersion, diskId)
raise Exception, errMsg
sql = "DELETE FROM ngas_files WHERE disk_id={0} AND file_id={1} AND file_version={2}"
self.query2(sql, args=(diskId, fileId, fileVersion))
# Create a File Removal Status Document.
if (self.getCreateDbSnapshot() and genSnapshot):
tmpFileObj = ngamsFileInfo.ngamsFileInfo().\
unpackSqlResult(dbFileInfo)
self.createDbFileChangeStatusDoc(hostId, NGAMS_DB_CH_FILE_DELETE,
[tmpFileObj])
# Now update the ngas_disks entry for the disk hosting the file.
if (dbDiskInfo):
newNumberOfFiles = (dbDiskInfo[ngamsDbCore.\
NGAS_DISKS_NO_OF_FILES] - 1)
if (newNumberOfFiles < 0): newNumberOfFiles = 0
newAvailMb = (dbDiskInfo[ngamsDbCore.NGAS_DISKS_AVAIL_MB] +
int(float(dbFileInfo[ngamsDbCore.\
NGAS_FILES_FILE_SIZE])/1e6))
newBytesStored = (dbDiskInfo[ngamsDbCore.\
NGAS_DISKS_BYTES_STORED] -
dbFileInfo[ngamsDbCore.NGAS_FILES_FILE_SIZE])
if (newBytesStored < 0): newBytesStored = 0
sql = "UPDATE ngas_disks SET number_of_files={0}, " +\
"available_mb={1}, bytes_stored={2} WHERE disk_id={3}"
self.query2(sql,
args=(newNumberOfFiles, newAvailMb, newBytesStored,
diskId))
self.triggerEvents()
return self
def deleteDiskInfo(self, diskId, delFileInfo=1):
"""
Delete a record for a certain disk in the NGAS DB.
CAUTION: IF THE DB USER WITH WHICH THERE IS LOGGED IN HAS PERMISSION
TO EXECUTE DELETE STATEMENTS, THE INFORMATION ABOUT THE
DISK IN THE NGAS DB WILL BE DELETED! THIS INFORMATION
CANNOT BE RECOVERED!!
diskId: ID of disk for which to delete the entry (string).
delFileInfo: If set to 1, the file information for the files
stored on the disk is deleted as well (integer/0|1).
Returns: Reference to object itself.
"""
T = TRACE()
fileInfoDbmName = None
fileInfoDbm = None
try:
# Get the information about the files on the disk (before this
# information is deleted).
if delFileInfo and self.getCreateDbSnapshot():
diskInfo = ngamsDiskInfo.ngamsDiskInfo().read(self, diskId)
fileInfoDbmName = self.genTmpFile('DISK_INFO')
fileInfoDbm = ngamsDbm.ngamsDbm(fileInfoDbmName,
cleanUpOnDestr=0,
writePerm=1)
fileCount = 0
for fileInfo in self.getFileInfoList(diskId, fetch_size=1000):
fileInfoDbm.add(str(fileCount), fileInfo)
fileCount += 1
fileInfoDbm.sync()
# Delete the disk info.
sql = "DELETE FROM ngas_disks WHERE disk_id={}"
self.query2(sql, args=(diskId, ))
# Delete file info if requested.
if delFileInfo:
sql = "DELETE FROM ngas_files WHERE disk_id={}"
self.query2(sql, args=(diskId, ))
# Create a File Removal Status Document.
if (self.getCreateDbSnapshot()):
op = NGAMS_DB_CH_FILE_DELETE
fileInfoDbm.initKeyPtr()
key, fileSqlInfo = fileInfoDbm.getNext()
while (key):
tmpFileObj = ngamsFileInfo.ngamsFileInfo().\
unpackSqlResult(fileSqlInfo)
self.createDbRemFileChangeStatusDoc(
diskInfo, tmpFileObj)
key, fileSqlInfo = fileInfoDbm.getNext()
self.triggerEvents(
[diskInfo.getDiskId(),
diskInfo.getMountPoint()])
return self
except Exception:
logger.exception("Error deleting disk info from DB")
raise
finally:
if fileInfoDbm:
del fileInfoDbm
if fileInfoDbmName:
rmFile(fileInfoDbmName)
def writeFileEntry(self,
hostId,
diskId,
filename,
fileId,
fileVersion,
format,
fileSize,
uncompressedFileSize,
compression,
ingestionDate,
ignore,
checksum,
checksumPlugIn,
fileStatus,
creationDate,
iotime,
ingestionRate,
genSnapshot=1,
updateDiskInfo=0):
"""
The method writes the information in connection with a file in the
NGAS DB. If an entry already exists for that file, it is updated
with the information contained in the File Info Object. Otherwise,
a new entry is created.
diskId Values for the columns in the ngas_disks
... table (use values returned from ngamsFileInfo).
genSnapshot: Generate a snapshot file (integer/0|1).
updateDiskInfo: Update automatically the disk info for the
disk hosting this file (integer/0|1).
Returns: Void.
"""
T = TRACE(5)
# Check if the entry already exists. If yes update it, otherwise
# insert a new element.
if ignore == -1:
ignore = 0
checksum = str(checksum) if checksum else None
ingDate = self.convertTimeStamp(ingestionDate)
creDate = self.convertTimeStamp(creationDate)
sql = []
vals = []
sql_str = None
if (self.fileInDb(diskId, fileId, fileVersion)):
# We only allow to modify a limited set of columns.
sql.append(("UPDATE ngas_files SET "
"file_name={}, format={}, file_size={}, "
"uncompressed_file_size={}, compression={}, "
"%s={}, checksum={}, checksum_plugin={}, "
"file_status={}, creation_date={}, io_time={}, "
"ingestion_rate={} WHERE file_id={} AND disk_id={}" %
(self._file_ignore_columnname, )))
vals = [filename, format, fileSize, uncompressedFileSize, compression,\
ignore, checksum, checksumPlugIn, fileStatus, creDate,\
int(iotime*1000), ingestionRate, fileId, diskId]
if int(fileVersion) != -1:
sql.append(" AND file_version={}")
vals.append(fileVersion)
dbOperation = NGAMS_DB_CH_FILE_UPDATE
sql_str = ''.join(sql)
else:
sql_str = (
"INSERT INTO ngas_files (disk_id, file_name, file_id,"
"file_version, format, file_size, uncompressed_file_size,"
" compression, ingestion_date, %s, checksum, "
"checksum_plugin, file_status, creation_date, io_time, "
"ingestion_rate) VALUES ({}, {}, {}, {}, {}, {}, {}, {},"
" {}, {}, {},{}, {}, {}, {}, {})" %
(self._file_ignore_columnname, ))
vals = (diskId, filename, fileId, fileVersion, format, fileSize,\
uncompressedFileSize, compression, ingDate, ignore,\
checksum, checksumPlugIn, fileStatus, creDate,\
int(iotime*1000), ingestionRate)
dbOperation = NGAMS_DB_CH_FILE_INSERT
self.query2(sql_str, args=vals)
# Update the Disk Info of the disk concerned if requested and
# if a new entry was added.
#
# Note: In case of an update the columns ngas_disks.avail_mb
# and ngas_disks.bytes_stored should in principle be
# updated according to the actual size of the new
# version of the file.
#print "writeFileEntry".upper(), updateDiskInfo, dbOperation
if (updateDiskInfo and (dbOperation == NGAMS_DB_CH_FILE_INSERT)):
self.updateDiskFileStatus(diskId, fileSize)
# Create a Temporary DB Change Snapshot Document if requested.
if (self.getCreateDbSnapshot() and genSnapshot):
tmpFileObj = ngamsFileInfo.ngamsFileInfo().\
setDiskId(diskId).setFilename(filename).setFileId(fileId).\
setFileVersion(fileVersion).setFormat(format).setFileSize(fileSize).\
setUncompressedFileSize(uncompressedFileSize).setCompression(compression).\
setIngestionDate(ingestionDate).setIgnore(ignore).setChecksum(checksum).\
setChecksumPlugIn(checksumPlugIn).setFileStatus(fileStatus).setCreationDate(creationDate).\
setIoTime(iotime).setIngestionRate(ingestionRate)
self.createDbFileChangeStatusDoc(hostId, dbOperation, [tmpFileObj])
del tmpFileObj
self.triggerEvents([diskId, None])
def startFile(self, filename):
MIMEMultipartHandler.startFile(self, filename)
logger.debug('Found file: %s', filename)
fileInfo = ngamsFileInfo.ngamsFileInfo()
fileInfo.setFileId(filename)
self._container.addFileInfo(fileInfo)