def _getFileInfo1(ngamsDbObj, diskId):
"""
Retrieve information about files supposedly cloned.
ngamsDbObj: NG/AMS DB object (ngamsDb).
diskId: ID of disk to check (string).
Returns: NG/AMS Cursor Object (ngamsDbCursor).
"""
info(5, "Entering _getFileInfo1() ...")
sqlQueryFormat = "SELECT nd.slot_id, nd.mount_point, nf.file_name, " +\
"nf.checksum, nf.checksum_plugin, nf.file_id, " +\
"nf.file_version, nf.file_size, nf.file_status, " +\
"nd.disk_id, nf.file_ignore, nd.host_id " +\
"FROM ngas_disks nd noholdlock, " +\
"ngas_files nf noholdlock " +\
"WHERE nf.disk_id='%s' AND nd.disk_id='%s'"
sqlQuery = sqlQueryFormat % (diskId, diskId)
# Create a cursor and perform the query.
ngamsDbObj._startTimer()
server, db, user, password = ngasUtilsLib.getDbPars()
curObj = ngamsDb.ngamsDb(server, db, user, password, 0).dbCursor(sqlQuery)
ngamsDbObj._storeRdTime()
info(5, "Leaving _getFileInfo1(). DB Time: " + ngamsDbObj.getDbTimeStr())
return curObj
def _getFileInfo2(ngamsDbObj, hostId, srcDisk):
"""
Get information about all files stored on the given host.
ngamsDbObj: NG/AMS DB Object (ngamsDb).
hostId: ID of host for which to query files (string).
srcDisk: Source disk, i.e., disk that was cloned (string).
Returns: NG/AMS Cursor Object (ngamsDbCursor).
"""
info(5, "Entering _getFileInfo2() ...")
sqlQueryFormat = "SELECT nd.slot_id, nd.mount_point, nf.file_name, " +\
"nf.checksum, nf.checksum_plugin, nf.file_id, " +\
"nf.file_version, nf.file_size, nf.file_status, " +\
"nd.disk_id, nf.file_ignore, nd.host_id " +\
"FROM ngas_disks nd noholdlock, " +\
"ngas_files nf noholdlock " +\
"WHERE nd.host_id='%s' " +\
"AND nf.disk_id=nd.disk_id AND nf.disk_id!='%s'"
sqlQuery = sqlQueryFormat % (hostId, srcDisk)
# Create a cursor and perform the query.
ngamsDbObj._startTimer()
server, db, user, password = ngasUtilsLib.getDbPars()
curObj = ngamsDb.ngamsDb(server, db, user, password, 0).dbCursor(sqlQuery)
ngamsDbObj._storeRdTime()
info(5, "Leaving_getFileInfo2 (). DB Time: " + ngamsDbObj.getDbTimeStr())
return curObj
def execute(optDic):
"""
Carry out the tool execution.
optDic: Dictionary containing the options (dictionary).
Returns: Void.
"""
info(4, "Entering execute() ...")
if (optDic["help"][NGAS_OPT_VAL]):
print correctUsage()
sys.exit(0)
# Update the Completed Flag for the disk in question.
if (optDic["set-uncompleted"][2]):
completed = 0
else:
completed = 1
server, db, user, password = ngasUtilsLib.getDbPars()
dbCon = ngamsDb.ngamsDb(server, db, user, password, 0)
sqlQuery = "UPDATE ngas_disks SET completed=%d WHERE disk_id='%s'" %\
(completed, optDic["disk-id"][2])
dbCon.query(sqlQuery)
# # Restart server to update NgasDiskInfo file.
# host = ngasUtilsLib.getParNgasRcFile(ngasUtilsLib.NGAS_RC_PAR_HOST)
# port = int(ngasUtilsLib.getParNgasRcFile(ngasUtilsLib.NGAS_RC_PAR_PORT))
# # IMPL: Check that the NGAS Server specified is running locally.
# msg = "Reinitialize the NGAS Node (%s:%d) (y/n) [n]?" % (host, port)
# restart = ngasUtilsLib.input(msg)
# if ((restart.upper() == "Y") or (restart.upper() == "YES")):
# info(1,"Reinitializing NGAS Server ...")
# ngasClient = ngamsPClient.ngamsPClient(host, port)
# stat = ngasClient.init()
# if (stat.getStatus() == NGAMS_SUCCESS):
# info(1,"NGAS Server successfully reinitialized")
# else:
# info(1,"NGAS Server could not be reinitialized. Error: %s" %\
# stat.getMessage())
# else:
# info(1,"NOTE: Update of disk status first complete when NGAS Server "+\
# "brought to OFFLINE!")
info(4, "Leaving execute()")
def _hideSrcDisks():
"""
Used to suppress data dir.
Returns: Void.
"""
prepNgasResFile(BASE_CFG_1)
server, db, user, password = ngasUtilsLib.getDbPars()
dbConObj = ngamsDb.ngamsDb(server, db, user, password, 0)
pathPat = "/tmp/ngamsTest/NGAS:8001/FitsStorage1-%s/saf/%s"
# Make data dir disappear, mark disk as unmounted in the DB.
for name in ("Main-1", "Rep-2"):
dirOrg = pathPat % (name, "2001-05-08")
dirTmp = pathPat % (name, ".2001-05-08")
diskInfo = dirOrg + "/../../NgasDiskInfo"
diskId = ngamsStatus.ngamsStatus().load(diskInfo, 1).\
getDiskStatusList()[0].getDiskId()
ngamsDiskInfo.\
ngamsDiskInfo().read(dbConObj, diskId).\
setHostId("dummy-node").write(dbConObj)
mvFile(dirOrg, dirTmp)
def checkCopies(diskId,
notifEmail):
"""
Check the total number of copies of the files found on the referred disk
in the NGAS DB. Generate a report indicating the number of copies found.
diskId: ID of disk for which to check files (string).
notifEmail: Comma separated list of recipients of the report
generated (string).
Returns: Void.
"""
# Open DB connection.
server, db, user, password = ngasUtilsLib.getDbPars()
dbCon = ngamsDb.ngamsDb(server, db, user, password, 0)
# Get the information about the files on the referenced disk.
info(0,"Retrieving info about files on disk: %s ..." % diskId)
dbCur = dbCon.getFileSummary1(hostId=None, diskIds=[diskId], fileIds=[],
ignore=0)
diskFileList = []
while (1):
tmpList = dbCur.fetch(500)
if (tmpList == []): break
diskFileList += tmpList
time.sleep(0.100)
if ((len(diskFileList) % 100) == 0):
sys.stdout.write(".")
sys.stdout.flush()
del dbCur
print ""
info(0,"Retrieved info about files on disk: %s" % diskId)
# Get all files found in the system with the given File ID/File Version.
globFileList = []
queryFileIds = []
count = 0
noOfFiles = len(diskFileList)
info(0,"Retrieving info about all File IDs/Versions in the system " +\
"on disk: %s ..." % diskId)
for fileInfo in diskFileList:
count += 1
queryFileIds.append(fileInfo[ngamsDb.SUM1_FILE_ID])
if (((len(queryFileIds) % 20) == 0) or (count == noOfFiles)):
dbCur = dbCon.getFileSummary1(None, [], queryFileIds, ignore=0)
while (1):
tmpList = dbCur.fetch(500)
if (tmpList == []): break
globFileList += tmpList
del dbCur
queryFileIds = []
time.sleep(0.010)
if ((len(globFileList) % 100) == 0):
sys.stdout.write(".")
sys.stdout.flush()
print ""
info(0,"Retrieved info about all File IDs/Versions in the system " +\
"on disk: %s" % diskId)
# Now, go through this list, and generate a dictionary with
# File ID/File Version as keys.
globFileDic = {}
for fileInfo in globFileList:
fileKey = ngamsLib.genFileKey(None, fileInfo[ngamsDb.SUM1_FILE_ID],
fileInfo[ngamsDb.SUM1_VERSION])
if (not globFileDic.has_key(fileKey)): globFileDic[fileKey] = []
globFileDic[fileKey].append(fileInfo)
# Order the list according to 1) Number of copies, 2) Alphabetically.
fileKeys = globFileDic.keys()
fileKeys.sort()
sortFileDic = {}
for fileKey in fileKeys:
fileInfoList = globFileDic[fileKey]
noOfCopies = len(fileInfoList)
if (not sortFileDic.has_key(noOfCopies)):
sortFileDic[noOfCopies] = {}
sortFileDic[noOfCopies][fileKey] = fileInfoList
# Go through the global file dictionary and check file for each
# File ID/File Version the requested information.
report = "FILE COPIES CHECK REPORT:\n\n"
report += "Disk ID: " + diskId + "\n\n"
format = "%-40s %-7s %-7s %-7s\n"
report += format % ("File ID", "Version", "Total", "Good")
report += 80 * "-" + "\n"
noKeys = sortFileDic.keys()
noKeys.sort()
for noKey in noKeys:
noKeyDic = sortFileDic[noKey]
fileKeys = noKeyDic.keys()
fileKeys.sort()
for fileKey in fileKeys:
totCopies = 0
goodCopies = 0
for fileInfo in noKeyDic[fileKey]:
totCopies += 1
if ((fileInfo[ngamsDb.SUM1_FILE_STATUS][0] == "0") and
(fileInfo[ngamsDb.SUM1_FILE_IGNORE] == 0)):
goodCopies += 1
fileId = fileInfo[ngamsDb.SUM1_FILE_ID]
fileVer = fileInfo[ngamsDb.SUM1_VERSION]
report += format % (fileId, str(fileVer), str(totCopies),
str(goodCopies))
if (len(noKeys)):
report += 80 * "-" + "\n\n"
else:
report += "No files found on the given disk!\n\n"
print "\n" + report
if (notifEmail):
report
ngasUtilsLib.sendEmail("ngasCheckFileCopies: " +\
"FILE COPIES CHECK REPORT (%s)" % diskId,
notifEmail, report)
def retireDisk(diskId,
force,
execute,
notifEmail):
"""
Remove file entries from the ngas_files DB, which are stored on disks
having the mount point column set to 'RETIRED'.
diskId: Disk ID of disk to retire (string).
execute: Execute the command. If set to 0, the command is executed
otherwise it is only checked if the command can be
executed (integer/0|1).
force: Even though there are less than 3 copies of each file on
the disk in the system or even if the disk is mounted
retire the disk (integer/0|1).
notifEmail: Comma separated list of email recipients that should be
informed about the actions carried out, or the actions
that would be carried out if executing the command
(integer/0|1).
Returns: Void.
"""
info(0,"Executing Retire Disk Procedure on disk with ID: %s" % diskId)
# Open DB connection.
info(3,"Open DB connection")
server, db, user, password = ngasUtilsLib.getDbPars()
dbCon = ngamsDb.ngamsDb(server, db, user, password, 0)
# Dummy server object, needed by one of the methods used ...
srvObj = ngamsServer.ngamsServer().setDb(dbCon)
# Get information about the disk.
info(3,"Get info about disk in question")
sqlDiskInfo = dbCon.getDiskInfoFromDiskId(diskId)
if (sqlDiskInfo == []):
raise Exception, "Disk ID given: %s not found in the NGAS DB!" % diskId
diskInfoObj = ngamsDiskInfo.ngamsDiskInfo().unpackSqlResult(sqlDiskInfo)
info(0,"Disk with ID: %s has Logical Name: %s" %\
(diskInfoObj.getDiskId(), diskInfoObj.getLogicalName()))
# Check if the disk is marked as mounted, in case yes, reject it.
if (diskInfoObj.getMounted()):
errMsg = "Disk with ID: %s is marked as mounted in the DB! "+\
"Rejecting request."
raise Exception, errMsg % diskId
# Check if there are at least three copies of each file.
if (not force):
basePath = "/tmp/.ngasRetireDisk"
checkCreatePath(basePath)
ngasUtilsLib.checkDelTmpDirs(basePath + "/*")
tmpPath = "/tmp/.ngasRetireDisk/" + str(time.time())
checkCreatePath(tmpPath)
dbFilePat = tmpPath + "/DBM"
lessCopyDbm, notRegDbm, allFilesDbm =\
ngamsRemUtils.checkFileCopiesAndReg(srvObj, 3, dbFilePat,
None, diskId, 1)
srvObj.getCfg().storeVal("NgamsCfg.Notification[1].SmtpHost",
"smtphost.hq.eso.org")
reqPropsObj = ngamsReqProps.ngamsReqProps().\
setCmd("ngasRetireDisk").\
addHttpPar("notif_email", notifEmail).\
addHttpPar("disk_id", diskId)
status = ngamsRemUtils._remStatErrReport(srvObj, reqPropsObj,
dbFilePat, lessCopyDbm,
notRegDbm, allFilesDbm,
diskId)
commands.getstatusoutput("rm -rf " + dbFilePat + "*")
if (status):
errMsg = "Cannot retire disk with ID: %s - one or more files " +\
"are not available in at least 3 copies!"
raise Exception, errMsg % diskId
# First query the files in question.
info(0,"Get list of files stored on disk: %s ..." %\
diskInfoObj.getLogicalName())
query = "SELECT nf.disk_id, file_id, file_version " +\
"FROM ngas_files nf, ngas_disks nd WHERE " +\
"nd.disk_id='%s' AND nf.disk_id='%s'"
query = query % (diskId, diskId)
statFileList = dbCon.query(query, 1)
info(0,"Got list of files stored on disk: %s" %\
diskInfoObj.getLogicalName())
# Back-up the files on the retired disk to the ngas_files_retired table.
if (execute):
info(0,"Insert files on Retired Disk into ngas_files_retired")
query = "INSERT INTO ngas_files_retired SELECT disk_id, " +\
"file_name, file_id, file_version, format, file_size, " +\
"uncompressed_file_size, compression, ingestion_date, " +\
"file_ignore, checksum, checksum_plugin, file_status, " +\
"creation_date FROM ngas_files WHERE disk_id='%s'"
query = query % diskId
dbCon.query(query, 1)
info(0,"Inserted files on Retired Disk into ngas_files_retired")
# Remove the entries from the DB.
if (execute):
info(0,"Remove entry from disk from DB ...")
query = "DELETE FROM ngas_files WHERE disk_id='%s'" % diskId
dbCon.query(query, 1)
info(0,"Removed entry from disk from DB")
# Set the mount point of the referenced disk to 'RETIRED'.
if (execute):
info(0,"Setting mount point of Retired Disk to 'RETIRED' ...")
query = "UPDATE ngas_disks SET mount_point='RETIRED' WHERE " +\
"disk_id='%s'"
query = query % diskId
dbCon.query(query, 1)
info(0,"Set mount point of Retired Disk to 'RETIRED'")
# Generate report.
if (notifEmail):
info(0,"Generate DISK RETIREMENT STATUS REPORT ...")
report = "DISK RETIREMENT STATUS REPORT - %s:\n\n" % diskId
if (not execute):
report += "Information for disk/files concerned if command " +\
"is executed.\n\n"
else:
report += "Information for disk/files concerned.\n\n"
report += "Date: %s\n" % PccUtTime.TimeStamp().getTimeStamp()
report += "NGAS Host: %s\n" % getHostName()
report += "Disk ID: %s\n\n" % diskId
tmpFormat = "%-32s %-32s %-12s\n"
report += tmpFormat % ("Disk ID", "File ID", "File Version")
report += tmpFormat % (32 * "-", 32 * "-", 12 * "-")
if (len(statFileList[0])):
for fileInfo in statFileList[0]:
report += tmpFormat % (fileInfo[0], fileInfo[1], fileInfo[2])
else:
report += "No files found on disk!\n"
report += 78 * "-" + "\n"
subject = "ngasRetireDisk: DISK RETIREMENT REPORT - %s" % diskId
ngasUtilsLib.sendEmail(subject, notifEmail, report)
info(0,"Generated DISK RETIREMENT STATUS REPORT")
info(0,"Finished Disk Retirement Procedure for disk: %s/%s" %\
(diskInfoObj.getDiskId(), diskInfoObj.getLogicalName()))
def changeDiskLabel(diskId, newLabel=None, execute=0):
"""
Change the Disk Label (ngas_disks.logical_name) of the disk referred
to by its Disk ID.
diskId: Disk ID of the disk (string).
newLabel: New label to allocate to the disk (string).
execute: Execute the change (integer/0|1).
Returns: Void.
"""
# Open DB connection.
server, db, user, password = ngasUtilsLib.getDbPars()
dbCon = ngamsDb.ngamsDb(server, db, user, password, 0)
# Now, loop and ask the user to enter the new label and to confirm when OK.
while (1):
sqlDiskInfo = dbCon.getDiskInfoFromDiskId(diskId)
if (not sqlDiskInfo):
msg = "Disk with ID: %s appears not to be known to this system!"
raise Exception, msg % diskId
diskInfoObj = ngamsDiskInfo.ngamsDiskInfo().\
unpackSqlResult(sqlDiskInfo)
orgDiskLabel = diskInfoObj.getLogicalName()
print "Present Disk Label: %s" % orgDiskLabel
if (newLabel):
newDiskLabel = newLabel
else:
newDiskLabel = ngasUtilsLib.input("Enter new Disk Label:")
if (sqlDiskInfo == []):
raise Exception, "Disk ID given: %s not found in the NGAS DB!" %\
diskId
print "\nChanging Disk Label for Disk with:"
print " - Disk ID: " + diskId
print " - Disk Label: " + orgDiskLabel
print " - New Disk Label: " + newDiskLabel + "\n"
if (execute):
break
else:
choice = ngasUtilsLib.input("Is this correct (Y/N) [N]?").upper()
if (choice == "Y"): break
print "Changing Disk Label from %s to %s ..." %\
(orgDiskLabel, newDiskLabel)
# Check if this disk label already has been allocated.
logNames = dbCon.getLogicalNamesMountedDisks(getHostName())
for logName in logNames:
if (logName == newDiskLabel):
raise Exception, "Disk Label: %s is already in use!" % newDiskLabel
diskInfoObj.setLogicalName(newDiskLabel).write(dbCon)
sqlDiskInfo = dbCon.getDiskInfoFromDiskId(diskId)
diskInfoObj = ngamsDiskInfo.ngamsDiskInfo().unpackSqlResult(sqlDiskInfo)
report = "DISK LABEL CHANGE REPORT:\n\n"
msg = "Changed Disk Label from %s to %s - new disk status:" %\
(orgDiskLabel, newDiskLabel)
report += msg + "\n\n"
print msg
msg = diskInfoObj.dumpBuf()
report += msg + "\n\n"
print msg
print("\nPut the given disk Online/reboot the host in " +\
"which the disk is inserted."
"\n\nCheck the disk info in the NGAS WEB Interfaces!\n")
notifEmail = ngasUtilsLib.\
getParNgasRcFile(ngasUtilsLib.NGAS_RC_PAR_NOTIF_EMAIL)
if (notifEmail):
ngasUtilsLib.sendEmail("ngasChangeDiskLabel: DISK LABEL CHANGE REPORT",
notifEmail, report)
def checkCloning(srcDiskId, autoClone):
"""
Function to check if all files registered for a given disk, have
been cloned onto disks in the system where this tool is executed.
srcDiskId: ID of the cloned disk (string).
autoClone: If 1 files not cloned will be automatically cloned
by the tool (integer/0|1).
Returns: Status message (string).
"""
if (autoClone):
portNo = ngasUtilsLib.getParNgasRcFile("NgasPort")
ngamsClient = ngamsPClient.ngamsPClient(getHostName(), portNo)
else:
ngamsClient = None
checkRepMain = 79 * "=" + "\n"
checkRepMain += "Clone Verification Report - Disk: " + srcDiskId + "\n\n"
checkRep = ""
server, db, user, password = ngasUtilsLib.getDbPars()
dbCon = ngamsDb.ngamsDb(server, db, user, password, 0)
test = 0
if (not test):
# Get information about files on the Source Disk.
diskInfo = dbCon.getDiskInfoFromDiskId(srcDiskId)
if (not diskInfo):
raise Exception, "Specified Disk ID: " + srcDiskId +\
" is unknown. Aborting."
if (diskInfo[ngamsDb.NGAS_DISKS_HOST_ID].strip() == getHostName()):
raise Exception, "Source disk specified Disk ID: %s" % srcDiskId+\
" is inserted in this node: %s" % getHostName()
srcFiles = []
if (diskInfo[ngamsDb.NGAS_DISKS_HOST_ID].strip() != ""):
srcHost = diskInfo[ngamsDb.NGAS_DISKS_HOST_ID].strip()
elif (diskInfo[ngamsDb.NGAS_DISKS_LAST_HOST_ID].strip() != ""):
srcHost = diskInfo[ngamsDb.NGAS_DISKS_LAST_HOST_ID].strip()
else:
srcHost = None
dbCur = _getFileInfo1(dbCon, srcDiskId)
while (1):
fileList = dbCur.fetch(200)
if (fileList == []): break
srcFiles += fileList
del dbCur
# Get information about files on the Target Host (= this host).
trgFiles = []
# NGAS Utils Functional Tests: Use special target host if requested.
dbCur = _getFileInfo2(dbCon, _getTargHost(), srcDiskId)
while (1):
fileList = dbCur.fetch(200)
if (fileList == []): break
trgFiles += fileList
del dbCur
else:
# We load this information from a file where we have dumped
# it previously to avoid too long execution time.
import ngasVerifyCloningFileInfo
srcFiles = ngasVerifyCloningFileInfo.getSrcFiles()
trgFiles = ngasVerifyCloningFileInfo.getTrgFiles()
# Build a dictionary with the target files with
# (<File ID>, <File Version>) as keys.
trgFileDic = {}
for fileInfo in trgFiles:
trgFileDic[(fileInfo[ngamsDb.SUM1_FILE_ID],
fileInfo[ngamsDb.SUM1_VERSION])] = fileInfo
# Now go through each source file and check if it is registered
# in the DB among the target files.
for fileInfo in srcFiles:
srcFileId = fileInfo[ngamsDb.SUM1_FILE_ID]
srcFileVer = fileInfo[ngamsDb.SUM1_VERSION]
key = (srcFileId, srcFileVer)
# Check if target file is present in the DB.
if (not trgFileDic.has_key(key)):
checkRep += "*** Missing target file in DB: " + str(
fileInfo) + "\n"
if (autoClone):
checkRep += cloneCheckFile(fileInfo[9], fileInfo[5],
fileInfo[6], dbCon, ngamsClient,
checkRep) + "\n"
continue
trgFileInfo = trgFileDic[key]
mtPt = trgFileInfo[ngamsDb.SUM1_MT_PT]
filename = trgFileInfo[ngamsDb.SUM1_FILENAME]
complTrgFilename = os.path.normpath(mtPt + "/" + filename)
msg = "*** Checking file: " + complTrgFilename
if (autoClone): msg = "\n" + msg
print msg
# 1. Check that the target file is physically present on the
# target disk.
if (not os.path.exists(complTrgFilename)):
checkRep += "Missing target file on disk: " + str(fileInfo) + "\n"
if (autoClone):
checkRep += cloneCheckFile(fileInfo[9], fileInfo[5],
fileInfo[6], dbCon, ngamsClient,
checkRep) + "\n"
continue
# 2. Check that the size is correct.
srcFileSize = fileInfo[ngamsDb.SUM1_FILE_SIZE]
trgFileSize = ngamsLib.getFileSize(complTrgFilename)
if (srcFileSize != trgFileSize):
checkRep += "Wrong size of target file: " + str(fileInfo) + "\n"
checkRep += " - Check file manually!" + "\n"
if (checkRep):
checkRepMain += checkRep
else:
checkRepMain += "No descrepancies found\n"
return checkRepMain
def cloneFileList(host, port, fileListName, notifEmail):
"""
Clone the files given in the file list, which is a Notification Email
Message from the NG/AMS Server.
host: Name of NGAS host to contact (string).
port: Port number used by NG/AMS Server (integer).
fileListName: File containing the Email Notification Message from
the NG/AMS Server about missing file copies (string).
notifEmail: Comma separated list of email recipients that should be
informed about the actions carried out, or the actions
that would be carried out if executing the command
(integer/0|1).
Returns: Void.
"""
fileRefList = loadFileList(fileListName)
ngamsClient = ngamsPClient.ngamsPClient(host, port)
server, db, user, password = ngasUtilsLib.getDbPars()
dbCon = ngamsDb.ngamsDb(server, db, user, password, 0)
report = "FILE CLONING REPORT:\n\n"
for fileInfo in fileRefList:
diskId = fileInfo[0]
fileId = fileInfo[1]
fileVer = fileInfo[2]
msg = "Cloning file: %s/%s/%s" % (str(diskId), str(fileId),
str(fileVer))
sys.stdout.write(msg)
tmpMsg = " - Status: "
sys.stdout.write(tmpMsg)
msg += tmpMsg
res = ngamsClient.clone(fileId, diskId, fileVer, wait=1)
if (res.getStatus() == NGAMS_FAILURE):
status = NGAMS_FAILURE
else:
# Wait for the command to finish.
#status = ngasVerifyCloning.waitCloneCmd(res.getRequestId(),
# ngamsClient, 60)
# Get the Disk ID of a disk in that system, different than the
# source disk, where the file is stored.
curObj = dbCon.getFileSummary1(host, [], [fileId])
tmpFileList = curObj.fetch(1000)
tmpDiskId = ""
for fileInfo in tmpFileList:
tmpDiskId = fileInfo[ngamsDb.SUM1_DISK_ID]
tmpFileVer = fileInfo[ngamsDb.SUM1_VERSION]
if (tmpDiskId != diskId): break
del curObj
if (tmpDiskId != diskId):
# Check that the file has been cloned.
res = ngamsClient.sendCmdGen(
host, port, NGAMS_CHECKFILE_CMD, 1, "",
[["host_id", host], ["disk_id", tmpDiskId],
["file_id", fileId], ["file_version", fileVer]])
if (res.getMessage().find("NGAMS_INFO_FILE_OK") != -1):
status = NGAMS_SUCCESS
else:
status = NGAMS_FAILURE
else:
status = "CHECK!!!"
sys.stdout.write(status + "\n")
msg += status + "\n"
report += msg
report += 100 * "-" + "\n"
print 100 * "-" + "\n"
if (notifEmail):
ngasUtilsLib.sendEmail("ngasCloneFileList: FILE CLONE REPORT",
notifEmail, report)
def ingestExtFiles(fileListFile, notifEmail):
"""
Ingest the list of files given in the file referred to.
The files are listed as:
<Disk ID> <File ID> [<File Version>]
...
fileListFile: Filename of file containing the file list (string).
notifEmail: List of email addresses to inform about the
execution of the discation procedure (string).
Returns: Void.
"""
fileInfoList = parseFileList(fileListFile)
# Open DB connection.
server, db, user, password = ngasUtilsLib.getDbPars()
dbCon = ngamsDb.ngamsDb(server, db, user, password, 0)
# Find distinct Disk IDs.
diskIdDic = {}
for fileInfo in fileInfoList:
diskIdDic[fileInfo[0]] = None
info(1, "Disk IDs: " + str(diskIdDic.keys()))
# Register disks referred to as external disks in the DB.
dbDiskDic = {}
for diskId in diskIdDic.keys():
if (not dbDiskDic.has_key(diskId)):
diskInfo = dbCon.getDiskInfoFromDiskId(diskId)
if (diskInfo != []):
diskInfoObj = ngamsDiskInfo.ngamsDiskInfo().\
unpackSqlResult(diskInfo)
else:
# Create a new entry for that disk in the DB.
info(1,"Creating new entry for external disk with ID: " +\
str(diskId))
diskInfoObj = ngamsDiskInfo.ngamsDiskInfo().\
setArchive("EXTERNAL").\
setDiskId(diskId).\
setLogicalName(diskId).\
setHostId("").\
setSlotId("").\
setMounted(0).\
setMountPoint("EXTERNAL").\
setNumberOfFiles(0).\
setAvailableMb(0).\
setBytesStored(0).\
setCompleted(1).\
setCompletionDateFromSecs(0).\
setType("EXTERNAL").\
setManufacturer("UNKNOWN").\
setInstallationDateFromSecs(time.time()).\
setChecksum(0).\
setTotalDiskWriteTime(0).\
setLastCheckFromSecs(0).\
setLastHostId("").\
setStorageSetId(diskId)
diskInfoObj.write(dbCon)
diskIdDic[diskId] = diskInfoObj
# Loop over the files and register them in the DB.
fileRegList = []
sys.stdout.write("Registering files ...")
fileCount = 0
for fileInfo in fileInfoList:
diskId = fileInfo[0]
fileId = fileInfo[1]
fileVer = fileInfo[2]
fileInfoObj = ngamsFileInfo.ngamsFileInfo().\
setDiskId(diskId).\
setFilename(fileId).\
setFileId(fileId).\
setFileVersion(fileVer).\
setFormat("").\
setFileSize(0).\
setUncompressedFileSize(0).\
setCompression("").\
setIngestionDateFromSecs(0).\
setIgnore(0).\
setChecksum("").\
setChecksumPlugIn("").\
setFileStatus(NGAMS_FILE_STATUS_OK).\
setCreationDateFromSecs(0).\
setTag("EXTERNAL")
fileInfoObj.write(getHostName(), dbCon, 0, 1)
fileRegList.append((diskId, fileId, fileVer))
time.sleep(0.050)
fileCount += 1
if ((fileCount % 10) == 0): sys.stdout.write(".")
sys.stdout.write("\n")
info(1, "Registered %d files" % fileCount)
report = genReport(fileRegList)
if (notifEmail):
ngasUtilsLib.sendEmail("ngasRegisterExtFiles: FILE REGISTRATION " +\
"REPORT", notifEmail, report)