def cleanUp(self):
"""
Close a possible open DB connection + remove the DB file from the disk.
Returns: Reference to object itself.
"""
if (self.__dbmObj): self.__dbmObj.sync()
rmFile(self.__dbmName)
return self
def createObjPickleFile(filename,
object):
"""
Create a file containing the pickled image of the object given.
filename: Name of pickle file (string).
object: Object to be pickled (<object>)
Returns: Void.
"""
logger.debug("createObjPickleFile() - creating pickle file %s ...", filename)
rmFile(filename)
with open(filename, "w") as pickleFo:
cPickle.dump(object, pickleFo)
def save(self, docType, schema, xmlDoc=None, critInfoNameList=[]):
"""
Save the NG/AMS Configuration into the connected XML document
or the one given in the input parameter.
xmlDoc: Name of XML document (string).
Returns: Reference to object itself.
"""
T = TRACE()
if (xmlDoc):
targetFile = xmlDoc
rmFile(targetFile)
else:
targetFile = self.getXmlDoc()
fo = open(targetFile, "w")
fo.write(self.genXmlDoc(docType, schema, critInfoNameList))
fo.close()
return self
def two_stage_pickle(mtdir, obj):
"""
Performs a two-stage writing of a new .pickle file under the cache directory
of the indicated mount directory. The file is first written with a .tmp
and later renamed to have a .pickle extension so the Janitor Thread doesn't
pick it up
until it's completed.
"""
dirname = os.path.join(mtdir, NGAMS_DB_CH_CACHE)
tmpfd, tmpfname = tempfile.mkstemp(".tmp", dir=dirname, text=False)
fd, fname = tempfile.mkstemp(".pickle", dir=dirname, text=False)
os.close(fd)
try:
with os.fdopen(tmpfd, "wb") as pickleFo:
cPickle.dump(obj, pickleFo, 1)
mvFile(tmpfname, fname)
except:
rmFile(tmpfname)
raise
def sendEmail(ngamsCfgObj,
smtpHost,
subject,
recList,
fromField,
dataRef,
contentType=None,
attachmentName=None,
dataInFile=0):
"""
Send an e-mail to the recipients specified.
ngamsCfgObj: Reference to object containing NG/AMS
configuration file (ngamsConfig).
smtpHost: Mail server to use for sending the mail (string).
subject: Subject of mail message (string).
recList: List containing recipients, e.g. [email protected] (string).
fromField: Name for the from field (string).
dataRef: Message to send or reference to file containing data
to send (string).
contentType: Mime-type of message (string).
attachmentName: Name of attachment in mail (string).
dataInFile: Used to make the sendmail tool send the data contained
in a file (integer).
Returns: Void.
"""
T = TRACE()
hdr = "Subject: " + subject + "\n"
if (contentType):
hdr += "Content-Type: " + contentType + "\n"
if (attachmentName):
hdr += "Content-Disposition: attachment; filename=" +\
attachmentName + "\n"
if (not dataInFile):
data = hdr + "\n" + dataRef
else:
# Prepare a file with the exact contents of the email.
data = genTmpFilename(ngamsCfgObj, "_NOTIF_EMAIL.tmp")
fo = open(data, "w")
fo.write(hdr + "\n")
foIn = open(dataRef)
while (1):
buf = foIn.read(16384)
if (buf == ""): break
fo.write(buf)
fo.close()
foIn.close()
for emailAdr in recList:
try:
server = ngamsSmtpLib.ngamsSMTP(smtpHost)
server.sendMail("From: " + fromField, "Bcc: " + emailAdr, data, [],
[], dataInFile)
except Exception, e:
if (dataInFile): rmFile(data)
errMsg = genLog("NGAMS_ER_EMAIL_NOTIF",
[emailAdr, fromField, smtpHost,
str(e)])
raise Exception(errMsg)
if (not dataInFile):
data = hdr + "\n" + dataRef
else:
# Prepare a file with the exact contents of the email.
data = genTmpFilename(ngamsCfgObj, "_NOTIF_EMAIL.tmp")
fo = open(data, "w")
fo.write(hdr + "\n")
foIn = open(dataRef)
while (1):
buf = foIn.read(16384)
if (buf == ""): break
fo.write(buf)
fo.close()
foIn.close()
for emailAdr in recList:
try:
server = ngamsSmtpLib.ngamsSMTP(smtpHost)
server.sendMail("From: " + fromField, "Bcc: " + emailAdr, data, [],
[], dataInFile)
except Exception, e:
if (dataInFile): rmFile(data)
errMsg = genLog("NGAMS_ER_EMAIL_NOTIF",
[emailAdr, fromField, smtpHost,
str(e)])
raise Exception(errMsg)
if (dataInFile): rmFile(data)
# EOF
def deleteDiskInfo(self, diskId, delFileInfo=1):
"""
Delete a record for a certain disk in the NGAS DB.
CAUTION: IF THE DB USER WITH WHICH THERE IS LOGGED IN HAS PERMISSION
TO EXECUTE DELETE STATEMENTS, THE INFORMATION ABOUT THE
DISK IN THE NGAS DB WILL BE DELETED! THIS INFORMATION
CANNOT BE RECOVERED!!
diskId: ID of disk for which to delete the entry (string).
delFileInfo: If set to 1, the file information for the files
stored on the disk is deleted as well (integer/0|1).
Returns: Reference to object itself.
"""
T = TRACE()
fileInfoDbmName = None
fileInfoDbm = None
try:
# Get the information about the files on the disk (before this
# information is deleted).
if delFileInfo and self.getCreateDbSnapshot():
diskInfo = ngamsDiskInfo.ngamsDiskInfo().read(self, diskId)
fileInfoDbmName = self.genTmpFile('DISK_INFO')
fileInfoDbm = ngamsDbm.ngamsDbm(fileInfoDbmName,
cleanUpOnDestr=0,
writePerm=1)
fileCount = 0
for fileInfo in self.getFileInfoList(diskId, fetch_size=1000):
fileInfoDbm.add(str(fileCount), fileInfo)
fileCount += 1
fileInfoDbm.sync()
# Delete the disk info.
sql = "DELETE FROM ngas_disks WHERE disk_id={}"
self.query2(sql, args=(diskId, ))
# Delete file info if requested.
if delFileInfo:
sql = "DELETE FROM ngas_files WHERE disk_id={}"
self.query2(sql, args=(diskId, ))
# Create a File Removal Status Document.
if (self.getCreateDbSnapshot()):
op = NGAMS_DB_CH_FILE_DELETE
fileInfoDbm.initKeyPtr()
key, fileSqlInfo = fileInfoDbm.getNext()
while (key):
tmpFileObj = ngamsFileInfo.ngamsFileInfo().\
unpackSqlResult(fileSqlInfo)
self.createDbRemFileChangeStatusDoc(
diskInfo, tmpFileObj)
key, fileSqlInfo = fileInfoDbm.getNext()
self.triggerEvents(
[diskInfo.getDiskId(),
diskInfo.getMountPoint()])
return self
except Exception:
logger.exception("Error deleting disk info from DB")
raise
finally:
if fileInfoDbm:
del fileInfoDbm
if fileInfoDbmName:
rmFile(fileInfoDbmName)
def dumpFileInfoCluster(self,
clusterName,
fileInfoDbmName=None,
useFileKey=False,
count=False):
"""
Dump the info for the files registered in the name space of the
referenced cluster.
Note, all files in the cluster are taken, also the ones marked
as bad or to be ignored.
clusterName: Name of cluster to consider (string).
fileInfoDbmName: Base name of the DBM in which the file info will be
stored. If not given, a name will be generated
automatically (string).
useFileKey: Use a file key (<File ID>_<Version>) as key as
opposed to just an integer key. NOTE: Multiple
occurrences of a given File ID/Version will only
appear once (boolean).
count: When useFileKey == True, if count is True, the
number of ocurrences of each File ID + Version
is counted and an entry added in the DBM:
<File Key>__COUNTER
- pointing to a counter indicating the number of
occurrences. Note, the usage of '__' in the name of
the counter for each file, means it will be skipped
when doing a ngamsDbm.getNext(), scanning through
the contents of the DBM (boolean).
Returns: Final name of the DBM DB containing the info about
the files (string).
"""
T = TRACE()
# Create a temporay File Info DBM.
if (not fileInfoDbmName):
fileInfoDbmName = self.genTmpFile("CLUSTER-FILE-INFO")
rmFile("%s*" % fileInfoDbmName)
# Get list of hosts in the cluster.
clusterHostList = self.getHostIdsFromClusterName(clusterName)
if (clusterHostList == []):
msg = "No hosts registered for cluster with name: %s" % clusterName
raise Exception, msg
clusterHostList = str(clusterHostList).strip()[1:-1]
if (clusterHostList[-1] == ","): clusterHostList = clusterHostList[:-1]
sql = ("SELECT %s FROM ngas_files nf WHERE disk_id IN "
"(SELECT disk_id FROM ngas_disks WHERE "
"host_id IN ({}) OR last_host_id IN ({}))") \
% ngamsDbCore.getNgasFilesCols(self._file_ignore_columnname)
try:
fileInfoDbm = ngamsDbm.ngamsDbm(fileInfoDbmName, 0, 1)
cursor = self.dbCursor(sql,
args=(clusterHostList, clusterHostList))
fileCount = 1
with cursor:
for fileInfo in cursor.fetch(1000):
if (not useFileKey):
fileInfoDbm.add(str(fileCount), fileInfo)
else:
fileId = fileInfo[ngamsDbCore.NGAS_FILES_FILE_ID]
fileVersion = fileInfo[ngamsDbCore.NGAS_FILES_FILE_VER]
fileKey = ngamsLib.genFileKey(None, fileId,
fileVersion)
if (count):
countKey = "%s__COUNTER" % fileKey
if (not fileInfoDbm.hasKey(countKey)):
fileInfoDbm.add(countKey, 0)
fileInfoDbm.add(countKey,
(fileInfoDbm.get(countKey) + 1))
fileInfoDbm.add(fileKey, fileInfo)
fileCount += 1
fileInfoDbm.sync()
del fileInfoDbm
except Exception, e:
rmFile(fileInfoDbmName)
msg = "dumpFileInfoCluster(): Failed in dumping file info. " +\
"Error: %s" % str(e)
raise Exception, msg
def __del__(self):
"""
Destructor method cleaning up.
"""
if (self.__dbmObj): self.__dbmObj.sync()
if (self.__cleanUpOnDestr): rmFile(self.__dbmName)
