def clone_check_file(disk_id, file_id, file_version, connection, ngas_client):
"""
Clone a file a check afterwards if it was successfully cloned
:param disk_id: ID of disk cloned (string)
:param file_id: ID of file cloned (string)
:param file_version: Version of file to check (integer)
:param connection: DB connection object (ngamsDb)
:param ngas_client: Initiated instance of NG/AMS P-Client Object (ngamsPClient)
:return: Updated Check Report (string)
"""
message = "\n-> Attempting to clone file: {:s}/{:s}/{:s}".format(
disk_id, file_id, file_version)
print(message)
message += " - status: "
result = ngas_client.clone(file_id, disk_id, file_version, wait=1)
if result.getStatus() == "FAILURE":
status = "FAILURE: " + str(result.get_message()) + "\n"
else:
# Check if file was really cloned
result = connection.getFileInfoFromFileIdHostId(
_get_target_host(), file_id, file_version)
if not result:
status = "FAILURE: File not cloned!\n"
else:
file_info = ngamsFileInfo.ngamsFileInfo().unpackSqlResult(result)
file_status_parameters = [["disk_id",
file_info.get_disk_id()],
["file_access", file_id],
["file_version", file_version]]
client = ngamsPClient.ngamsPClient(getHostName(),
ngas_client.getPort())
result = client.status(file_status_parameters)
status = result.get_message() + "\n"
return message + status
def locateArchiveFile(srvObj,
fileId,
fileVersion=-1,
diskId="",
hostId="",
reqPropsObj=None,
include_compression=False):
"""
Locate the file indicated by the File ID. Returns a list containing
the necessary information for retrieving the file:
[<Location>, <File Host>, <IP Address>, <Port No>, <Mount Point>,
<Filename>, <File ID>, <File Version>, <Mime-Type>]
- whereby:
<Location> = Location of the file (NGAMS_HOST_LOCAL,
NGAMS_HOST_CLUSTER, NGAMS_HOST_DOMAIN,
NGAMS_HOST_REMOTE).
<File Host> = Host ID of host to be contacted to get access to the
file.
<IP Address> = IP Address of host to be contacted to get access to the
file.
<Port No> = Port number used by the NG/AMS Server.
<Mount Point> = Mount point at which the file is residing.
<Filename> = Name of file relative to mount point.
<File ID> = ID of file.
<File Version> = Version of file.
<Mime-Type> = Mime-type of file (as registered in NGAS).
srvObj: Reference to NG/AMS server class object (ngamsServer).
fileId: File ID of file to locate (string).
fileVersion: Version of the file (integer).
diskId: ID of the disk hosting the file (string).
hostId: ID of the host where the file is located (string).
reqPropsObj: Request Property object to keep track of actions done during
the request handling (ngamsReqProps|None).
Returns: List with information about file location (list).
"""
# Get a list with the candidate files matching the query conditions.
res = srvObj.getDb().getFileInfoFromFileId(fileId,
fileVersion,
diskId,
ignore=0,
dbCursor=False)
# r[-2] is the host_id, r[-1] is the mount point
all_info = []
for r in res:
file_info = ngamsFileInfo.ngamsFileInfo().unpackSqlResult(r)
all_info.append((file_info, r[-2], r[-1]))
return _locateArchiveFile(srvObj, fileId, fileVersion, diskId, hostId,
reqPropsObj, all_info, include_compression)
def _encFileInfo2Obj(dbConObj, dbSnapshot, encFileInfoDic):
"""
Convert an encoded file info from the snapshot into an NG/AMS File
Info Object.
dbConObj: DB connection object (ngamsDb).
dbSnapshot: Open DB object (bsddb).
encFileInfoDic: Dictionary containing the encoded file information
(dictionary).
Returns: NG/AMS File Info Object (ngamsFileInfo).
"""
sqlFileInfo = []
#for n in range (ngamsDbCore.NGAS_FILES_CREATION_DATE + 1):
for n in range(ngamsDbCore.NGAS_FILES_CONTAINER_ID + 1):
sqlFileInfo.append(None)
idxKeys = encFileInfoDic.keys()
for idx in idxKeys:
kid = NGAMS_SN_SH_ID2NM_TAG + six.b(str(idx))
if (not dbSnapshot.has_key(kid)):
logger.warning("dbSnapshot has no key '%s', is it corrupted?",
str(kid))
return None
colName = _readDb(dbSnapshot, kid)
sqlFileInfoIdx = dbConObj.getNgasFilesMap()[colName]
sqlFileInfo[sqlFileInfoIdx] = encFileInfoDic[idx]
tmpFileInfoObj = ngamsFileInfo.ngamsFileInfo().unpackSqlResult(sqlFileInfo)
return tmpFileInfoObj
def test_FileVerHandling_01(self):
"""
Synopsis:
Test that REMFILE removes proper File Version.
Description:
The purpose of the test is to verify that if several versions of a
file are found in the archive, the proper version is deleted.
Expected Result:
The chosen version of the file, should be selected and should be
removed.
Test Steps:
- Start server.
- Archive a file 3 times into the archive.
- Clone the file onto another disk.
- Remove version 2 of the cloned file.
Remarks:
...
Test Data:
...
"""
_, dbObj = self.prepExtSrv(cfgProps=(('NgamsCfg.Db[1].Snapshot',
"0"), ))
for _ in range(3):
self.archive("src/SmallFile.fits")
diskId1 = self.ngas_disk_id("FitsStorage1/Main/1")
fileId = "TEST.2001-05-08T15:25:00.123"
fileVer = 2
self.get_status(NGAMS_CLONE_CMD, pars=[["disk_id", diskId1]])
diskId2 = self.ngas_disk_id("FitsStorage2/Main/3")
filePath = self.ngas_path("FitsStorage2-Main-3/saf/" +\
"2001-05-08/2/TEST.2001-05-08T15:25:00.123.fits.gz")
for execute in [0, 1]:
httpPars = [["disk_id", diskId2], ["file_id", fileId],
["file_version", fileVer], ["execute", execute]]
self.get_status(NGAMS_REMFILE_CMD, pars=httpPars)
fileInfo = ngamsFileInfo.ngamsFileInfo()
try:
fileInfo.read(getHostName(), dbObj, fileId, fileVer, diskId2)
except:
pass
if (execute == 0):
if (not os.path.exists(filePath)):
self.fail("File removed unexpectedly")
else:
if (os.path.exists(filePath)):
self.fail("File not removed as expected")
refStatFile = "ref/ngamsRemFileCmdTest_test_" +\
"FileVerHandling_01_0%d_ref" % (execute + 3)
msg = "Incorrect status for REMFILE Command/execution"
self.assert_status_ref_file(refStatFile, fileInfo, msg=msg)
def test_RegisterCmd_1(self):
"""
Synopsis:
REGISTER Command/register single file compl. path.
Description:
Test handling of the REGISTER Command under normal circumstances.
Expected Result:
The REGISTER Command should be accepted by the server and should
be executed successfully.
Test Steps:
- Start server.
- Copy file onto NGAS Disk.
- Submit REGISTER Command requesting to register the file copied over
(wait=1).
- Check response from the server that the request was successfully
executed.
- Check the DB info for the registered file.
Remarks:
...
"""
_, dbObj = self.prepExtSrv()
srcFile = "src/SmallFile.fits"
tmpSrcFile = "/tmp/ngamsTest/NGAS/" +\
"FitsStorage2-Main-3/saf/test/SmallFile.fits"
checkCreatePath(os.path.dirname(tmpSrcFile))
shutil.copy(srcFile, tmpSrcFile)
tmpStatFile = sendExtCmd(8888, NGAMS_REGISTER_CMD,
[["path", tmpSrcFile]])
refStatFile = "ref/ngamsRegisterCmdTest_test_RegisterCmd_1_ref"
self.checkFilesEq(refStatFile, tmpStatFile,
"Incorrect status returned for REGISTER command")
diskId = "tmp-ngamsTest-NGAS-FitsStorage2-Main-3"
filePrefix = "ngamsRegisterCmdTest_test_RegisterCmd_1"
fileInfoRef = "ref/" + filePrefix + "_FileInfo_ref"
fileInfoTmp = "tmp/" + filePrefix + "_FileInfo_tmp"
fileId = "TEST.2001-05-08T15:25:00.123"
startTime = time.time()
host_id = getHostName() + ":8888"
while ((time.time() - startTime) < 10):
tmpFileRes = dbObj.getFileInfoFromFileIdHostId(
host_id, fileId, 1, diskId)
if (tmpFileRes): break
if not tmpFileRes:
self.fail(
"Couldn't get fileInfo result from database within 10 seconds")
tmpFileObj = ngamsFileInfo.ngamsFileInfo().unpackSqlResult(tmpFileRes)
saveInFile(fileInfoTmp, filterDbStatus1(tmpFileObj.dumpBuf()))
self.checkFilesEq(fileInfoRef, fileInfoTmp,
"Incorrect info in DB for registered file")
def test_get_file_info_list_with_wildcards(self):
"""Double-check that wildcards in fileId work"""
disk_info = ngamsDiskInfo.ngamsDiskInfo()
disk_info.setDiskId('disk-id')
disk_info.write(self.db)
file_info = ngamsFileInfo.ngamsFileInfo()
file_info.setDiskId('disk-id')
file_info.setFileId('file-id')
file_info.write('host-id', self.db, genSnapshot=0)
res = list(self.db.getFileInfoList('disk-id', fileId="*"))
self.assertEqual(1, len(res))
def receiveData(srvObj,
reqPropsObj,
httpRef):
"""
Receive the data in connection with the Rearchive Request.
For a description of the parameters: Check handleCmd().
Returns: Tuple with File Info Object for the file to be rearchived and
Disk Info Object for the selected target disk
(tuple/(fileInfo, ngamsDiskInfo)).
"""
# Note, this algorithm does not implement support for back-log buffering
# for speed optimization reasons.
# Unpack the file information contained in the HTTP header
# (NGAS-File-Info).
encFileInfo = reqPropsObj.getHttpHdr(NGAMS_HTTP_HDR_FILE_INFO)
if (not encFileInfo):
msg = "Error. Must provide NGAS File Information " +\
"(RFC 3548 encoded) in HTTP header NGAS-File-Info"
raise Exception, msg
fileInfoXml = base64.b64decode(encFileInfo)
fileInfoObj = ngamsFileInfo.ngamsFileInfo().unpackXmlDoc(fileInfoXml)
# Find the most suitable target Disk Set.
trgDiskInfoObj = None
diskExList = []
while (True):
try:
tmpTrgDiskInfo = ngamsDiskUtils.\
findTargetDisk(srvObj.getHostId(),
srvObj.getDb(), srvObj.getCfg(),
reqPropsObj.getMimeType(),
sendNotification = 0,
diskExemptList = diskExList,
reqSpace = reqPropsObj.getSize())
if (srvObj.getDb().fileInDb(tmpTrgDiskInfo.getDiskId(),
fileInfoObj.getFileId(),
fileInfoObj.getFileVersion())):
diskExList.append(tmpTrgDiskInfo.getDiskId())
else:
trgDiskInfoObj = tmpTrgDiskInfo
break
except Exception, e:
msg = "Error locating target disk for REARCHIVE Command. Error: %s"
msg = msg % str(e)
raise Exception, msg
def test_RegisterCmd_1(self):
"""
Synopsis:
REGISTER Command/register single file compl. path.
Description:
Test handling of the REGISTER Command under normal circumstances.
Expected Result:
The REGISTER Command should be accepted by the server and should
be executed successfully.
Test Steps:
- Start server.
- Copy file onto NGAS Disk.
- Submit REGISTER Command requesting to register the file copied over
(wait=1).
- Check response from the server that the request was successfully
executed.
- Check the DB info for the registered file.
Remarks:
...
"""
_, db_obj = self.prepExtSrv()
status = self.copy_and_register()
msg = "Incorrect status returned for REGISTER command"
ref_stat_file = "ref/ngamsRegisterCmdTest_test_RegisterCmd_1_ref"
self.assert_status_ref_file(ref_stat_file, status, msg=msg)
disk_id = self.ngas_disk_id("FitsStorage2/Main/3")
file_prefix = "ngamsRegisterCmdTest_test_RegisterCmd_1"
file_info_ref = "ref/" + file_prefix + "_FileInfo_ref"
file_id = "TEST.2001-05-08T15:25:00.123"
start_time = time.time()
host_id = getHostName() + ":8888"
while (time.time() - start_time) < 10:
tmp_file_res = db_obj.getFileInfoFromFileIdHostId(
host_id, file_id, 1, disk_id)
if tmp_file_res: break
if not tmp_file_res:
self.fail(
"Couldn't get fileInfo result from database within 10 seconds")
tmp_file_obj = ngamsFileInfo.ngamsFileInfo().unpackSqlResult(
tmp_file_res)
msg = "Incorrect info in DB for registered file"
self.assert_status_ref_file(file_info_ref, tmp_file_obj, msg=msg)
def rearchive(self, fileUri, fileInfoXml, pars=[]):
"""
Send a Re-archive Request to the associated NG/AMS Server asking to
have the file specified by the URI archived. This can either
be a Re-archive Push Request or a Re-archive Pull Request.
fileUri: URI of file to archive (string).
fileInfoXml: NG/AMS XML File Information for the source file used
as reference (string/XML).
pars: Extra parameters to submit with the request. Must be
in the format:
[[<Par>, <Val>], [<Par>, <Val>], ...] (list).
Returns: NG/AMS Status object (ngamsStatus).
"""
baseName = os.path.basename(fileUri)
logger.info("Re-archiving file with URI: %s", baseName)
pars = list(pars)
if is_known_pull_url(fileUri):
tmpFileInfo = ngamsFileInfo.ngamsFileInfo().\
unpackXmlDoc(fileInfoXml)
encFileInfo = base64.b64encode(fileInfoXml)
pars.append([NGAMS_HTTP_PAR_FILENAME, fileUri])
httpHdrs = {
NGAMS_HTTP_HDR_FILE_INFO: encFileInfo,
NGAMS_HTTP_HDR_CONTENT_TYPE: tmpFileInfo.getFormat()
}
res = self.get_status(NGAMS_REARCHIVE_CMD,
pars=pars,
hdrs=httpHdrs)
else:
msg = "Rearchive Push is not yet supported!"
raise Exception(msg)
return res
def _registerExec(srvObj,
fileListDbmName,
tmpFilePat,
diskInfoDic,
reqPropsObj=None):
"""
Register the files listed in the File List DBM (ngamsDbm), which
match the mime-type(s) either specified in the 'mimeType' parameter,
or if this is not specified, which match all the mime-types specified
in the configuration file.
When the registration procedure has been executed, the function sends
an Email Notification message indicating which files were registered
if the HTTP parameter 'notif_email' is given.
The functions creates a File Info Objects per file handled and
writes this in a temporary file, which is a DBM file. The
keys in this DB is simply the file number in the sequence of files
handled, pointing to a pickled ngamsFileInfo object.
Each of the File Info Objects indicates if the file was registered or not.
This is done by setting the tag of the File Info Object to one of the
following values:
REGISTERED: The file was successfully registered in the NGAS DB
FAILED: The file was selected for registration but could not
be properly registered because of inconsistencies.
The status will be of the format: 'FAILED[: <reason>]'.
REJECTED: A file found under the specified path directory was
not accepted for cloning, usually because the mime-type
was not correct. The status will be of the format:
'REJECTED[: <reason>]'.
Note, that registration is comparable to archiving of files. For that
reason a DAPI must be provided for each type of file that should be
registered. If this is not fullfilled, the file registration will
fail.
Only files stored on one of the NGAS disks configured in the
configuration file, are considered. Files stored in other locations
are ignored.
srvObj: Instance of NG/AMS Server object (ngamsServer).
fileListDbmName: Name of a DBM containing the information
about the files to be registered. Each element in the
list is referred to by a key, which is a number.
These points to a pickled list for each file containing
the following information:
[<Filename>, <Disk ID>, <Mime-Type>]
The information for each disk concerned is also contained
in the DB referred to by its Disk ID and by its mount
point. The data is a pickled instance of the ngamsDiskInfo
class (string).
tmpFilePat: Pattern for temporary files used during the registration
process (string).
diskInfoDic: Dictionary with Disk IDs as keys pointing to the info
about the disk (dictionary/ngamsDiskInfo).
reqPropsObj: If an NG/AMS Request Properties Object is given, the
Request Status will be updated as the request is carried
out (ngamsReqProps).
Returns: Void.
"""
emailNotif = 0
if (reqPropsObj):
if (reqPropsObj.hasHttpPar("notif_email")):
emailNotif = 1
# Create the temporary BSD DB to contain the information for the
# Email Notification Message.
if (emailNotif):
regDbmName = tmpFilePat + "_NOTIF_EMAIL"
regDbm = ngamsDbm.ngamsDbm(regDbmName, writePerm=1)
# Open the DBM containing the list of files to (possibly) register.
fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm=1)
# Want to parse files in alphabetical order.
# TODO: Portatibility issue. Try to avoid UNIX shell commands for sorting.
tmpFileList = tmpFilePat + "_FILE_LIST"
rmFile(tmpFileList)
with open(tmpFileList, "wb") as fo:
fileListDbm.initKeyPtr()
while (1):
dbmKey, fileInfo = fileListDbm.getNext()
if (not dbmKey): break
fo.write(dbmKey + b"\n")
sortFileList = tmpFilePat + "_SORT_FILE_LIST"
rmFile(sortFileList)
shellCmd = "sort %s > %s" % (tmpFileList, sortFileList)
stat, out, err = ngamsCore.execCmd(shellCmd)
if (stat != 0):
raise Exception("Error executing command: %s. Error: %s, %s" %\
(shellCmd, str(out), str(err)))
rmFile(tmpFileList)
# Go through each file in the list, check if the mime-type is among the
# ones, which apply for registration. If yes try to register the file
# by invoking the corresponding DAPI on the file.
fileRegCount = 0
fileFailCount = 0
fileRejectCount = 0
regTimeAccu = 0.0
fileCount = 0
fo = open(sortFileList)
run = 1
while (run):
reg_start = time.time()
dbmKey = fo.readline()
if (dbmKey.strip() == ""):
run = 0
continue
fileInfo = fileListDbm.get(dbmKey[0:-1])
filename = fileInfo[0]
diskId = fileInfo[1]
mimeType = fileInfo[2]
# Register the file. Check first, that exactly this file is
# not already registered. In case it is, the file will be rejected.
regPi = srvObj.getCfg().register_plugins[mimeType]
logger.debug("Plugin found for %s: %s", mimeType, regPi)
params = ngamsPlugInApi.parseRawPlugInPars(regPi.pars)
tmpReqPropsObj = ngamsReqProps.ngamsReqProps().\
setMimeType(mimeType).\
setStagingFilename(filename).\
setTargDiskInfo(diskInfoDic[diskId]).\
setHttpMethod(NGAMS_HTTP_GET).\
setCmd(NGAMS_REGISTER_CMD).\
setSize(os.path.getsize(filename)).\
setFileUri(filename).\
setNoReplication(1)
tmpFileObj = ngamsFileInfo.ngamsFileInfo()
try:
# Invoke Registration Plug-In.
piName = regPi.name
plugInMethod = loadPlugInEntryPoint(piName)
piRes = plugInMethod(srvObj, tmpReqPropsObj, params)
del tmpReqPropsObj
# Check if this file is already registered on this disk. In case
# yes, it is not registered again.
files = srvObj.db.getFileSummary1(srvObj.getHostId(),
[piRes.getDiskId()],
[piRes.getFileId()])
fileRegistered = 0
for tmpFileInfo in files:
tmpMtPt = tmpFileInfo[ngamsDbCore.SUM1_MT_PT]
tmpFilename = tmpFileInfo[ngamsDbCore.SUM1_FILENAME]
tmpComplFilename = os.path.normpath(tmpMtPt + "/" +\
tmpFilename)
if (tmpComplFilename == filename):
fileRegistered = 1
break
if (fileRegistered):
fileRejectCount += 1
tmpMsgForm = "REJECTED: File with File ID/Version: %s/%d " +\
"and path: %s is already registered on disk " +\
"with Disk ID: %s"
tmpMsg = tmpMsgForm % (piRes.getFileId(),
piRes.getFileVersion(), filename,
piRes.getDiskId())
logger.warning(tmpMsg + ". File is not registered again.")
if (emailNotif):
tmpFileObj.\
setDiskId(diskId).setFilename(filename).\
setTag(tmpMsg)
regDbm.addIncKey(tmpFileObj)
if (reqPropsObj):
reqPropsObj.incActualCount(1)
ngamsHighLevelLib.stdReqTimeStatUpdate(
srvObj, reqPropsObj, regTimeAccu)
regTimeAccu += time.time() - reg_start
fileCount += 1
continue
# Calculate checksum. We maintain the old name for backwards
# compatibility
crc_variant = srvObj.cfg.getCRCVariant()
if crc_variant == ngamsFileUtils.CHECKSUM_CRC32_INCONSISTENT:
crc_variant = 'ngamsGenCrc32'
checksum = ngamsFileUtils.get_checksum(65536, filename,
crc_variant) or ''
# Move file and update information about file in the NGAS DB.
mvFile(filename, piRes.getCompleteFilename())
ngamsArchiveUtils.updateFileInfoDb(srvObj, piRes, checksum,
crc_variant)
ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), piRes)
ngamsLib.makeFileReadOnly(piRes.getCompleteFilename())
if (emailNotif):
uncomprSize = piRes.getUncomprSize()
ingestDate = time.time()
creDateSecs = getFileCreationTime(filename)
tmpFileObj.\
setDiskId(diskId).\
setFilename(filename).\
setFileId(piRes.getFileId()).\
setFileVersion(piRes.getFileVersion()).\
setFormat(piRes.getFormat()).\
setFileSize(piRes.getFileSize()).\
setUncompressedFileSize(uncomprSize).\
setCompression(piRes.getCompression()).\
setIngestionDate(ingestDate).\
setIgnore(0).\
setChecksum(checksum).\
setChecksumPlugIn(crc_variant).\
setFileStatus(NGAMS_FILE_STATUS_OK).\
setCreationDate(creDateSecs).\
setTag("REGISTERED")
fileRegCount += 1
# If running as a cache archive, update the Cache New Files DBM
# with the information about the new file.
if (srvObj.getCachingActive()):
fileVer = fio.getFileVersion()
ngamsCacheControlThread.addEntryNewFilesDbm(
srvObj, diskId, piRes.getFileId(), fileVer, filename)
# Generate a confirmation log entry.
msg = genLog("NGAMS_INFO_FILE_REGISTERED", [
filename,
piRes.getFileId(),
piRes.getFileVersion(),
piRes.getFormat()
])
time.sleep(0.005)
regTime = time.time() - reg_start
msg = msg + ". Time: %.3fs." % (regTime)
logger.info(msg, extra={'to_syslog': 1})
except Exception as e:
errMsg = genLog("NGAMS_ER_FILE_REG_FAILED", [filename, str(e)])
logger.error(errMsg)
if (emailNotif):
tmpFileObj.\
setDiskId(diskId).setFilename(filename).\
setTag(errMsg)
fileFailCount += 1
regTime = time.time() - reg_start
# TODO (rtobar, 2016-01): Why don't we raise an exception here?
# Otherwise the command appears as successful on the
# client-side
# Add the file information in the registration report.
if (emailNotif): regDbm.addIncKey(tmpFileObj)
# Update request status time information.
regTimeAccu += regTime
if (reqPropsObj):
reqPropsObj.incActualCount(1)
ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj,
regTimeAccu)
fileCount += 1
fo.close()
rmFile(sortFileList)
if (emailNotif): regDbm.sync()
del fileListDbm
rmFile(fileListDbmName + "*")
# Final update of the Request Status.
if (reqPropsObj):
if (reqPropsObj.getExpectedCount() and reqPropsObj.getActualCount()):
complPercent = (100.0 * (float(reqPropsObj.getActualCount()) /
float(reqPropsObj.getExpectedCount())))
else:
complPercent = 100.0
reqPropsObj.setCompletionPercent(complPercent, 1)
reqPropsObj.setCompletionTime(1)
srvObj.updateRequestDb(reqPropsObj)
# Send Register Report with list of files cloned to a possible
# requestor(select) of this.
if (emailNotif):
xmlStat = 0
if (xmlStat):
# Create an instance of the File List Class, used to store the
# Registration Report.
regStat = ngamsFileList.\
ngamsFileList("FILE_REGISTRATION_STATUS",
"Status report for file " +\
"registration")
# Loop over the file objects in the BSD DB and add these
# in the status object.
regDbm.initKeyPtr()
while (1):
key, tmpFileObj = regDbm.getNext()
if (not key): break
regStat.addFileInfoObj(tmpFileObj)
# Set overall status of registration procedure.
regStat.setStatus("Files Found: " + str(fileCount + 1) + ", "+\
"Files Registered: " + str(fileRegCount) +\
", " +\
"Files Failed: " + str(fileFailCount) + ", " +\
"Files Rejected: " + str(fileRejectCount))
status = srvObj.genStatus(NGAMS_SUCCESS,
"REGISTER command status report").\
addFileList(regStat)
statRep = status.genXmlDoc()
statRep = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, statRep)
mimeType = NGAMS_XML_MT
else:
# Generate a 'simple' ASCII report.
statRep = tmpFilePat + "_NOTIF_EMAIL.txt"
fo = open(statRep, "w")
if (reqPropsObj):
path = reqPropsObj.getHttpPar("path")
else:
path = "-----"
if (fileCount):
timePerFile = (regTimeAccu / fileCount)
else:
timePerFile = 0
tmpFormat = "REGISTER STATUS REPORT:\n\n" +\
"==Summary:\n\n" +\
"Date: %s\n" +\
"NGAS Host: %s\n" +\
"Search Path: %s\n" +\
"Total Number of Files: %d\n" +\
"Number of Registered Files: %d\n" +\
"Number of Failed Files: %d\n" +\
"Number of Rejected Files: %d\n" +\
"Total processing time (s): %.3f\n" +\
"Handling time per file (s): %.3f\n\n" +\
"==File List:\n\n"
fo.write(tmpFormat % (toiso8601(), srvObj.getHostId(), path,
fileCount, fileRegCount, fileFailCount,
fileRejectCount, regTimeAccu, timePerFile))
tmpFormat = "%-80s %-32s %-3s %-10s\n"
fo.write(tmpFormat %\
("Filename", "File ID", "Ver", "Status"))
fo.write(tmpFormat % (80 * "-", 32 * "-", 3 * "-", 10 * "-"))
regDbm.initKeyPtr()
while (1):
key, tmpFileObj = regDbm.getNext()
if (not key): break
mtPt = diskInfoDic[tmpFileObj.getDiskId()].getMountPoint()
filename = os.path.normpath(mtPt + "/" +\
tmpFileObj.getFilename())
line = tmpFormat %\
(filename, tmpFileObj.getFileId(),
str(tmpFileObj.getFileVersion()),
tmpFileObj.getTag())
fo.write(line)
fo.write(128 * "-")
fo.write("\n\n==END\n")
fo.close()
mimeType = NGAMS_TEXT_MT
# Send out the status report.
emailAdrList = reqPropsObj.getHttpPar("notif_email").split(",")
attachmentName = "RegisterStatusReport"
if (reqPropsObj.hasHttpPar("path")):
attachmentName += "-" + reqPropsObj.getHttpPar("path").\
replace("/", "_")
ngamsNotification.notify(srvObj.host_id,
srvObj.cfg,
NGAMS_NOTIF_INFO,
"REGISTER STATUS REPORT",
statRep,
recList=emailAdrList,
force=1,
contentType=mimeType,
attachmentName=attachmentName)
del regDbm
rmFile(regDbmName + "*")
rmFile(statRep)
# Generate final status log + exit.
if (fileCount > 0):
timePerFile = (regTimeAccu / fileCount)
else:
timePerFile = 0.0
msg = "Registration procedure finished processing Register Request - " + \
"terminating. Files handled: %d. Total time: %.3fs. " + \
"Average time per file: %.3fs."
logger.debug(msg, fileCount, regTimeAccu, timePerFile)
def _clone(srvObj, diskId, fileId, fileVersion, targetDiskId, reqPropsObj,
httpRef, tmpFilePat):
"""
Internal function used by ngamsCloneCmd.clone() to carry out the
cloning. See documentation for ngamsCloneCmd.clone().
"""
targetDiskId = targetDiskId.strip()
logger.debug("Handling file cloning with parameters - File ID: %s -" + \
"Disk ID: %s - File Version: %s - Target Disk ID: |%s|",
fileId, diskId, str(fileVersion), targetDiskId)
if (((fileId == "") and (diskId == "") and (fileVersion != -1))
or ((fileId == "") and (diskId == "") and (fileVersion == -1))):
errMsg = genLog("NGAMS_ER_CMD_SYNTAX",
[NGAMS_CLONE_CMD, "File Id: " + fileId +\
", Disk ID: " + diskId +\
", File Version: " + str(fileVersion)])
raise Exception, errMsg
# If Disk ID, File ID and File Version are given, execute a quick cloning.
try:
fileVersion = int(fileVersion)
except:
pass
if (False and diskId and fileId and (fileVersion > 0)):
_cloneExplicit(srvObj, reqPropsObj, diskId, fileId, fileVersion,
targetDiskId)
logger.info("Successfully handled command CLONE")
return
# Handling cloning of more files.
cloneListDbm = None
cloneListDbmName = tmpFilePat + "_CLONE_INFO_DB"
try:
# Get information about candidate files for cloning.
files = srvObj.db.getFileInfoFromFileId(fileId,
fileVersion,
diskId,
ignore=0,
order=0,
dbCursor=False)
if not files:
msg = genLog("NGAMS_ER_CMD_EXEC",
[NGAMS_CLONE_CMD, "No files for cloning found"])
raise Exception(msg)
# Convert to tuple of file info object plus extra info
# This is how the code expects this information to come, so we need to
# keep the format unless we change the bulk of the code
# f[-2] is the host id, f[-1] is the mount point
all_info = []
for f in files:
all_info.append((ngamsFileInfo.ngamsFileInfo().unpackSqlResult(f),
f[-2], f[-1]))
# Create a BSD DB with information about files to be cloned.
rmFile(cloneListDbmName + "*")
cloneListDbm = ngamsDbm.ngamsDbm(cloneListDbmName,
cleanUpOnDestr=0,
writePerm=1)
cloneDbCount = 0
if fileId != "" and (diskId != "" or fileVersion == -1):
# Take only the first element
cloneListDbm.add("0", all_info[0])
cloneDbCount = 1
else:
# Take all the files.
for tmpFileInfo in all_info:
cloneListDbm.add(str(cloneDbCount), tmpFileInfo)
cloneDbCount += 1
except Exception:
if (cloneListDbm): del cloneListDbm
rmFile(cloneListDbmName + "*")
raise
logger.debug("Found: %d file(s) for cloning ...", cloneDbCount)
del cloneListDbm
# Check available amount of disk space.
cloneCheckDiskSpace(srvObj, cloneListDbmName, tmpFilePat, targetDiskId)
# Initialize Request Status parameters.
if (reqPropsObj):
reqPropsObj.\
setCompletionPercent(0, 1).\
setExpectedCount(cloneDbCount, 1).\
setActualCount(0, 1)
srvObj.updateRequestDb(reqPropsObj)
# Wait until CLONE Command has finished, or send a reply before cloning?
async = 'async' in reqPropsObj and int(reqPropsObj['async'])
if async:
# Send intermediate reply if the HTTP Reference object is given
# whenever send an auto reply now.
logger.debug("CLONE command accepted - generating immediate " +
"confimation reply to CLONE command")
status = srvObj.genStatus(NGAMS_SUCCESS,
"Accepted CLONE command for execution").\
setReqStatFromReqPropsObj(reqPropsObj).\
setActualCount(0)
# Do the actual cloning in a thread
args = (srvObj, cloneListDbmName, tmpFilePat, targetDiskId,
reqPropsObj, None)
thrName = NGAMS_CLONE_THR + threading.current_thread().getName()
cloneThread = threading.Thread(None, _cloneThread, thrName, args)
cloneThread.setDaemon(0)
cloneThread.start()
else:
# Carry out the cloning (directly in this thread) and send reply
# when this is done.
_cloneExec(srvObj, cloneListDbmName, tmpFilePat, targetDiskId,
reqPropsObj)
msg = "Successfully handled command CLONE"
logger.debug(msg)
status = srvObj.genStatus(NGAMS_SUCCESS, msg).\
setReqStatFromReqPropsObj(reqPropsObj).setActualCount(0)
rmFile(cloneListDbmName + "*")
# Send reply if possible.
if (httpRef):
xmlStat = status.genXmlDoc(0, 0, 0, 1, 0)
xmlStat = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, xmlStat)
httpRef.send_data(xmlStat, NGAMS_XML_MT)
def _cloneExplicit(srvObj, reqPropsObj, diskId, fileId, fileVersion,
targetDiskId):
"""
Execute CLONE Command, where the source Disk ID, File ID and File Version
are specified. Is much faster than a normal CLONE Command when an explicit
file is specified.
srvObj: Reference to instance of Server Object (ngamsServer).
fileInfoObj: File info object with info about the file
diskId: ID of disk hosting the file to be cloned (string).
fileId: ID of file to clone (string).
fileVersion: Version of file to clone (integer).
targetDiskId: ID of target disk (string).
Returns: Void.
"""
T = TRACE(1)
# Resolve the location of the file to clone.
location, hostId, ipAddress, portNo, mountPoint, filename,\
fileVersion, mimeType =\
ngamsFileUtils.quickFileLocate(srvObj, reqPropsObj, fileId,
diskId=diskId,
fileVersion=fileVersion)
# Read also the entire file info (unfortunately).
srcFileInfo = ngamsFileInfo.ngamsFileInfo().read(srvObj.getHostId(),
srvObj.getDb(), fileId,
fileVersion, diskId)
# Determine target disk.
if (targetDiskId == ""):
# Try to find a disk not hosting already a file with that
# ID + version.
diskExemptList = [diskId]
while (1):
trgDiskInfo = ngamsDiskUtils.\
findTargetDisk(srvObj.getHostId(),
srvObj.getDb(), srvObj.getCfg(),
mimeType, 1, diskExemptList)
# Check if a file with that ID + version is already
# stored on the selected Target Disk.
if (srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(), fileId,
fileVersion)):
# This file is already stored on the given disk.
# Add to the exempt list.
diskExemptList.append(trgDiskInfo.getDiskId())
else:
# OK, this disk should be OK, stop looking for a
# suitable Target Disk.
break
else:
trgDiskInfo = ngamsDiskInfo.ngamsDiskInfo().\
read(srvObj.getDb(), targetDiskId)
slotId = trgDiskInfo.getSlotId()
storageSetId = srvObj.getCfg().getStorageSetFromSlotId(slotId).\
getStorageSetId()
trgDiskInfo.setStorageSetId(storageSetId)
# Don't accept to clone onto the same disk (this would meann overwriting).
if (trgDiskInfo.getDiskId() == diskId):
err = "Source and target files are identical"
msg = "Failed in cloning file with ID: " + fileId +\
"/Version: " + str(fileVersion) +\
" on disk with ID: " + diskId +\
" on host: " + hostId + ". Reason: " + err
raise Exception, msg
# Receive the file into the staging filename.
tmpReqPropsObj = ngamsReqProps.ngamsReqProps()
tmpReqPropsObj.setMimeType(mimeType)
stagingFilename = ngamsHighLevelLib.genStagingFilename(
srvObj.getCfg(), tmpReqPropsObj, trgDiskInfo, fileId)
try:
quickLocation = False
if (reqPropsObj.hasHttpPar("quick")):
quickLocation = int(reqPropsObj.getHttpPar("quick"))
# Receive the data into the Staging File using the urllib.
if (srvObj.getHostId() != hostId):
# Example: http://host:7777/RETRIEVE?disk_id=%s&"
# file_id=id&file_version=1
fileUrl = "http://%s:%s/RETRIEVE?disk_id=%s&file_id=%s&" +\
"file_version=%s"
fileUrl = fileUrl % (ipAddress, str(portNo), diskId, fileId,
str(fileVersion))
# If CLONE?quick specified, we try to retrieve the file via the
# RETRIEVE?quick_location method.
quickFileUrl = fileUrl
if (reqPropsObj.hasHttpPar("quick")):
if (int(reqPropsObj.getHttpPar("quick"))):
quickFileUrl = fileUrl + "&quick_location=1"
# Check if host is suspended, if yes, wake it up.
if (srvObj.getDb().getSrvSuspended(hostId)):
logger.debug("Clone Request - Waking up suspended " +\
"NGAS Host: %s", hostId)
ngamsSrvUtils.wakeUpHost(srvObj, hostId)
else:
# TODO: a time-bomb waiting to explode....
fileUrl = "file:" + mtPt + "/" + filename
logger.debug("Receiving file via URI: %s into staging filename: %s",
fileUrl, stagingFilename)
# We try up to 5 times to retrieve the file in case a problem is
# encountered during cloning.
for attempt in range(5):
try:
if (attempt == 0):
filename, headers = urllib.urlretrieve(
quickFileUrl, stagingFilename)
else:
filename, headers = urllib.urlretrieve(
fileUrl, stagingFilename)
_checkFile(srvObj, srcFileInfo, stagingFilename, headers, True)
# If we get to this point the transfer was (probably) OK.
break
except Exception, e:
rmFile(stagingFilename)
errMsg = "Problem occurred while cloning file "+\
"via URL: " + fileUrl + " - Error: " + str(e)
if (attempt < 4):
errMsg += " - Retrying in 5s ..."
logger.error(errMsg)
time.sleep(0.5)
else:
raise Exception, errMsg
# We simply copy the file into the same destination as the
# source file (but on another disk).
targPathName = os.path.dirname(srcFileInfo.getFilename())
targFilename = os.path.basename(srcFileInfo.getFilename())
complTargPath = os.path.normpath(trgDiskInfo.getMountPoint() +\
"/" + targPathName)
checkCreatePath(complTargPath)
complFilename = os.path.normpath(complTargPath + "/" + targFilename)
mvTime = mvFile(stagingFilename, complFilename)
ngamsLib.makeFileReadOnly(complFilename)
# Update status for new file in the DB.
newFileInfo = srcFileInfo.clone().setDiskId(trgDiskInfo.getDiskId()).\
setCreationDate(getFileCreationTime(complFilename))
fileExists = srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(), fileId,
fileVersion)
newFileInfo.write(srvObj.getHostId(), srvObj.getDb())
# Update status for the Target Disk in DB + check if the disk is
# completed.
if (fileExists): mvTime = 0
dummyDapiStatObj = ngamsDapiStatus.ngamsDapiStatus().\
setDiskId(trgDiskInfo.getDiskId()).\
setFileExists(fileExists).\
setFileSize(srcFileInfo.getFileSize()).\
setIoTime(mvTime)
ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), dummyDapiStatObj)
ngamsArchiveUtils.checkDiskSpace(srvObj, trgDiskInfo.getDiskId())
# If running as a cache archive, update the Cache New Files DBM
# with the information about the new file.
if (srvObj.getCachingActive()):
ngamsCacheControlThread.addEntryNewFilesDbm(
srvObj, diskId, fileId, fileVersion, filename)
# Generate a confirmation log entry.
msg = genLog("NGAMS_INFO_FILE_CLONED",
[fileId, fileVersion, diskId, hostId])
logger.info(msg, extra={'to_syslog': True})
def checkDbChangeCache(srvObj, diskId, diskMtPt, stopEvt):
"""
The function merges the information in the DB Change Snapshot Documents
in the DB cache area on the disk concerned, into the Main DB Snapshot
Document in a safe way which prevents that any information is lost.
srvObj: Reference to NG/AMS server class object (ngamsServer).
diskId: ID for disk (string).
diskMtPt: Mount point of the disk, e.g. '/NGAS/disk1' (string).
Returns: Void.
"""
if (not srvObj.getCfg().getDbSnapshot()): return
if (not _updateSnapshot(srvObj.getCfg())): return
snapshotDbm = None
try:
snapshotDbm = _openDbSnapshot(srvObj.getCfg(), diskMtPt)
if (snapshotDbm == None):
return
# Remove possible, old /<mt pt>/.db/cache/*.xml snapshots.
# TODO: Remove when it can be assumed that all old XML snapshots have
# been removed.
rmFile(os.path.normpath(diskMtPt + "/" + NGAMS_DB_CH_CACHE + "/*.xml"))
# Update the Status document with the possibly new entries.
# TODO: Potential memory bottleneck. Use 'find > file' as for
# REGISTER Command.
dbCacheFilePat = os.path.normpath("%s/%s/*.%s" %\
(diskMtPt, NGAMS_DB_CH_CACHE,
NGAMS_PICKLE_FILE_EXT))
# Sort files by their creation date, to ensure we apply
# the DB changes in the order they were generated
tmpCacheFiles = glob.glob(dbCacheFilePat)
tmpCacheFiles.sort(key=lambda x: os.stat(x).st_ctime)
cacheStatObj = None
count = 0
fileCount = 0
noOfCacheFiles = len(tmpCacheFiles)
start = time.time()
for cacheFile in tmpCacheFiles:
checkStopJanitorThread(stopEvt)
if os.lstat(cacheFile)[6] == 0:
os.remove(
cacheFile) # sometimes there are pickle files with 0 size.
# we don't want to stop on them
continue
cacheStatObj = ngamsLib.loadObjPickleFile(cacheFile)
if isinstance(cacheStatObj, list):
# A list type in the Temporary DB Snapshot means that the
# file has been removed.
cacheStatList = cacheStatObj
tmpFileInfoObjList = [ngamsFileInfo.ngamsFileInfo().\
setDiskId(cacheStatList[0]).\
setFileId(cacheStatList[1]).\
setFileVersion(cacheStatList[2])]
operation = NGAMS_DB_CH_FILE_DELETE
elif (isinstance(cacheStatObj, ngamsFileInfo.ngamsFileInfo)):
tmpFileInfoObjList = [cacheStatObj]
operation = cacheStatObj.getTag()
else:
# Assume a ngamsFileList object.
cacheFileListObj = cacheStatObj.getFileListList()[0]
tmpFileInfoObjList = cacheFileListObj.getFileInfoObjList()
operation = cacheFileListObj.getComment()
# Loop over the files in the temporary snapshot.
for tmpFileInfoObj in tmpFileInfoObjList:
fileKey = ngamsDbm._ensure_binary(_genFileKey(tmpFileInfoObj))
fileInfoList = tmpFileInfoObj.genSqlResult()
encFileInfoDic = _encFileInfo(srvObj.getDb(), snapshotDbm,
fileInfoList)
if ((operation == NGAMS_DB_CH_FILE_INSERT)
or (operation == NGAMS_DB_CH_FILE_UPDATE)):
_addInDbm(snapshotDbm, fileKey, encFileInfoDic)
tmpFileInfoObj.write(srvObj.getHostId(), srvObj.getDb(), 0)
elif (operation == NGAMS_DB_CH_FILE_DELETE):
if (snapshotDbm.has_key(fileKey)): del snapshotDbm[fileKey]
_delFileEntry(srvObj.getHostId(), srvObj.getDb(),
tmpFileInfoObj)
else:
# Should not happen.
pass
del cacheStatObj
# Sleep if not last iteration (or if only one file).
fileCount += 1
if (fileCount < noOfCacheFiles): time.sleep(0.010)
# Synchronize the DB.
count += 1
if (count == 100):
snapshotDbm.sync()
checkStopJanitorThread(stopEvt)
count = 0
# Clean up, delete the temporary File Remove Status Document.
snapshotDbm.sync()
for cacheFile in tmpCacheFiles:
rmFile(cacheFile)
totTime = time.time() - start
tmpMsg = "Handled DB Snapshot Cache Files. Mount point: %s. " +\
"Number of Cache Files handled: %d."
args = [diskMtPt, fileCount]
if (fileCount):
tmpMsg += "Total time: %.3fs. Time per file: %.3fs."
args += (totTime, (totTime / fileCount))
logger.debug(tmpMsg, *args)
finally:
if snapshotDbm:
snapshotDbm.close()
def test_FileVerHandling_01(self):
"""
Synopsis:
Test that REMFILE removes proper File Version.
Description:
The purpose of the test is to verify that if several versions of a
file are found in the archive, the proper version is deleted.
Expected Result:
The chosen version of the file, should be selected and should be
removed.
Test Steps:
- Start server.
- Archive a file 3 times into the archive.
- Clone the file onto another disk.
- Remove version 2 of the cloned file.
Remarks:
...
Test Data:
...
"""
_, dbObj = self.prepExtSrv(cfgProps=(('NgamsCfg.Db[1].Snapshot',
"0"), ))
client = sendPclCmd()
for n in range(3):
stat = client.archive("src/SmallFile.fits")
self.assertEquals('SUCCESS', stat.getStatus())
diskId1 = "tmp-ngamsTest-NGAS-FitsStorage1-Main-1"
fileId = "TEST.2001-05-08T15:25:00.123"
fileVer = 2
status = client.get_status(NGAMS_CLONE_CMD,
pars=[["disk_id", diskId1]])
self.assertEquals('SUCCESS', status.getStatus())
diskId2 = "tmp-ngamsTest-NGAS-FitsStorage2-Main-3"
filePath = "/tmp/ngamsTest/NGAS/FitsStorage2-Main-3/saf/" +\
"2001-05-08/2/TEST.2001-05-08T15:25:00.123.fits.gz"
for execute in [0, 1]:
httpPars = [["disk_id", diskId2], ["file_id", fileId],
["file_version", fileVer], ["execute", execute]]
stat = client.get_status(NGAMS_REMFILE_CMD, pars=httpPars)
self.assertStatus(stat)
fileInfo = ngamsFileInfo.ngamsFileInfo()
try:
fileInfo.read(getHostName(), dbObj, fileId, fileVer, diskId2)
except:
pass
if (execute == 0):
if (not os.path.exists(filePath)):
self.fail("File removed unexpectedly")
else:
if (os.path.exists(filePath)):
self.fail("File not removed as expected")
refStatFile = "ref/ngamsRemFileCmdTest_test_" +\
"FileVerHandling_01_0%d_ref" % (execute + 3)
tmpStatFile = saveInFile(None, filterDbStatus1(fileInfo.dumpBuf()))
self.checkFilesEq(
refStatFile, tmpStatFile,
"Incorrect status for REMFILE Command/execution")
def _clone(srvObj,
diskId,
fileId,
fileVersion,
targetDiskId,
reqPropsObj,
httpRef,
tmpFilePat):
"""
Internal function used by ngamsCloneCmd.clone() to carry out the
cloning. See documentation for ngamsCloneCmd.clone().
"""
targetDiskId = targetDiskId.strip()
logger.debug("Handling file cloning with parameters - File ID: %s -" + \
"Disk ID: %s - File Version: %s - Target Disk ID: |%s|",
fileId, diskId, str(fileVersion), targetDiskId)
if not fileId and not diskId:
errMsg = genLog("NGAMS_ER_CMD_SYNTAX",
[NGAMS_CLONE_CMD, "File Id: " + fileId +\
", Disk ID: " + diskId +\
", File Version: " + str(fileVersion)])
raise Exception(errMsg)
# Get candidate files for cloning.
# We convert to tuple of file info object plus extra info because
# this is how the code expects this information to come, so we need to
# keep the format unless we change the bulk of the code
# f[-2] is the host id, f[-1] is the mount point
all_info = [(ngamsFileInfo.ngamsFileInfo().unpackSqlResult(f), f[-2], f[-1])
for f in srvObj.db.getFileInfoFromFileId(
fileId, fileVersion, diskId, ignore=0, order=0, order2=1,
dbCursor=False)]
if not all_info:
msg = genLog("NGAMS_ER_CMD_EXEC",
[NGAMS_CLONE_CMD, "No files for cloning found"])
raise Exception(msg)
# Take only the first element in these cases
if fileId != "" and (diskId != "" or fileVersion == -1):
all_info = [all_info[0]]
cloneDbCount = len(all_info)
cloneListDbmName = tmpFilePat + "_CLONE_INFO_DB"
rmFile(cloneListDbmName + "*")
try:
ngamsDbm.enumerate_to_dbm(cloneListDbmName, all_info)
except Exception:
rmFile(cloneListDbmName + "*")
raise
logger.debug("Found: %d file(s) for cloning ...", cloneDbCount)
# Check available amount of disk space.
cloneCheckDiskSpace(srvObj, cloneListDbmName, tmpFilePat, targetDiskId)
# Initialize Request Status parameters.
if (reqPropsObj):
reqPropsObj.\
setCompletionPercent(0, 1).\
setExpectedCount(cloneDbCount, 1).\
setActualCount(0, 1)
srvObj.updateRequestDb(reqPropsObj)
# Wait until CLONE Command has finished, or send a reply before cloning?
is_async = 'async' in reqPropsObj and int(reqPropsObj['async'])
if is_async:
# Send intermediate reply if the HTTP Reference object is given
# whenever send an auto reply now.
logger.debug("CLONE command accepted - generating immediate " +
"confimation reply to CLONE command")
status = srvObj.genStatus(NGAMS_SUCCESS,
"Accepted CLONE command for execution").\
setReqStatFromReqPropsObj(reqPropsObj).\
setActualCount(0)
# Do the actual cloning in a thread
args = (srvObj, cloneListDbmName, tmpFilePat, targetDiskId,
reqPropsObj, None)
thrName = NGAMS_CLONE_THR + threading.current_thread().getName()
cloneThread = threading.Thread(None, _cloneThread, thrName, args)
cloneThread.daemon = False
cloneThread.start()
else:
# Carry out the cloning (directly in this thread) and send reply
# when this is done.
_cloneExec(srvObj, cloneListDbmName, tmpFilePat, targetDiskId,
reqPropsObj)
msg = "Successfully handled command CLONE"
logger.debug(msg)
status = srvObj.genStatus(NGAMS_SUCCESS, msg).\
setReqStatFromReqPropsObj(reqPropsObj).setActualCount(0)
rmFile(cloneListDbmName + "*")
# Send reply if possible.
if (httpRef):
xmlStat = status.genXmlDoc(0, 0, 0, 1, 0)
xmlStat = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, xmlStat)
httpRef.send_data(six.b(xmlStat), NGAMS_XML_MT)
def ingestExtFiles(fileListFile, notifEmail):
"""
Ingest the list of files given in the file referred to.
The files are listed as:
<Disk ID> <File ID> [<File Version>]
...
fileListFile: Filename of file containing the file list (string).
notifEmail: List of email addresses to inform about the
execution of the discation procedure (string).
Returns: Void.
"""
fileInfoList = parseFileList(fileListFile)
# Open DB connection.
server, db, user, password = ngasUtilsLib.getDbPars()
dbCon = ngamsDb.ngamsDb(server, db, user, password, 0)
# Find distinct Disk IDs.
diskIdDic = {}
for fileInfo in fileInfoList:
diskIdDic[fileInfo[0]] = None
info(1, "Disk IDs: " + str(diskIdDic.keys()))
# Register disks referred to as external disks in the DB.
dbDiskDic = {}
for diskId in diskIdDic.keys():
if (not dbDiskDic.has_key(diskId)):
diskInfo = dbCon.getDiskInfoFromDiskId(diskId)
if (diskInfo != []):
diskInfoObj = ngamsDiskInfo.ngamsDiskInfo().\
unpackSqlResult(diskInfo)
else:
# Create a new entry for that disk in the DB.
info(1,"Creating new entry for external disk with ID: " +\
str(diskId))
diskInfoObj = ngamsDiskInfo.ngamsDiskInfo().\
setArchive("EXTERNAL").\
setDiskId(diskId).\
setLogicalName(diskId).\
setHostId("").\
setSlotId("").\
setMounted(0).\
setMountPoint("EXTERNAL").\
setNumberOfFiles(0).\
setAvailableMb(0).\
setBytesStored(0).\
setCompleted(1).\
setCompletionDateFromSecs(0).\
setType("EXTERNAL").\
setManufacturer("UNKNOWN").\
setInstallationDateFromSecs(time.time()).\
setChecksum(0).\
setTotalDiskWriteTime(0).\
setLastCheckFromSecs(0).\
setLastHostId("").\
setStorageSetId(diskId)
diskInfoObj.write(dbCon)
diskIdDic[diskId] = diskInfoObj
# Loop over the files and register them in the DB.
fileRegList = []
sys.stdout.write("Registering files ...")
fileCount = 0
for fileInfo in fileInfoList:
diskId = fileInfo[0]
fileId = fileInfo[1]
fileVer = fileInfo[2]
fileInfoObj = ngamsFileInfo.ngamsFileInfo().\
setDiskId(diskId).\
setFilename(fileId).\
setFileId(fileId).\
setFileVersion(fileVer).\
setFormat("").\
setFileSize(0).\
setUncompressedFileSize(0).\
setCompression("").\
setIngestionDateFromSecs(0).\
setIgnore(0).\
setChecksum("").\
setChecksumPlugIn("").\
setFileStatus(NGAMS_FILE_STATUS_OK).\
setCreationDateFromSecs(0).\
setTag("EXTERNAL")
fileInfoObj.write(getHostName(), dbCon, 0, 1)
fileRegList.append((diskId, fileId, fileVer))
time.sleep(0.050)
fileCount += 1
if ((fileCount % 10) == 0): sys.stdout.write(".")
sys.stdout.write("\n")
info(1, "Registered %d files" % fileCount)
report = genReport(fileRegList)
if (notifEmail):
ngasUtilsLib.sendEmail("ngasRegisterExtFiles: FILE REGISTRATION " +\
"REPORT", notifEmail, report)
def handleCmd(srvObj,
reqPropsObj,
httpRef):
"""
Handle STATUS command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler object
(ngamsHttpRequestHandler).
Returns: Void.
"""
status = ngamsStatus.ngamsStatus()
status.setDate(toiso8601()).\
setVersion(getNgamsVersion()).setHostId(srvObj.getHostId()).\
setStatus(NGAMS_SUCCESS).\
setMessage("Successfully handled command STATUS").\
setState(srvObj.getState()).setSubState(srvObj.getSubState())
reqPropsObjRef = reqPropsObj
# Get the information requested.
diskId = ""
fileId = ""
fileVersion = ""
configurationFile = ""
fileAccess = ""
hostId = ""
requestId = ""
dbTime = ""
dbTimeReset = ""
fileList = ""
fileListId = ""
maxElements = 100000
if (reqPropsObj.hasHttpPar("disk_id")):
diskId = reqPropsObj.getHttpPar("disk_id")
if (reqPropsObj.hasHttpPar("file_id")):
fileId = reqPropsObj.getHttpPar("file_id")
if (reqPropsObj.hasHttpPar("file_version")):
fileVersion = int(reqPropsObj.getHttpPar("file_version"))
if (reqPropsObj.hasHttpPar("configuration_file")):
configurationFile = reqPropsObj.getHttpPar("configuration_file")
if (reqPropsObj.hasHttpPar("file_access")):
fileAccess = reqPropsObj.getHttpPar("file_access")
if (reqPropsObj.hasHttpPar("host_id")):
hostId = reqPropsObj.getHttpPar("host_id")
if (reqPropsObj.hasHttpPar("max_elements")):
maxElements = int(reqPropsObj.getHttpPar("max_elements"))
if (reqPropsObj.hasHttpPar("request_id")):
requestId = reqPropsObj.getHttpPar("request_id")
if (reqPropsObj.hasHttpPar("db_time")):
dbTime = True
if (reqPropsObj.hasHttpPar("db_time_reset")):
dbTimeReset = True
if (reqPropsObj.hasHttpPar("flush_log")):
# in the past this called flushLog()
# do we really want to keep that functionality? I doubt it
pass
if (reqPropsObj.hasHttpPar("file_list")):
fileList = int(reqPropsObj.getHttpPar("file_list"))
if (reqPropsObj.hasHttpPar("file_list_id")):
fileListId = reqPropsObj.getHttpPar("file_list_id")
if (reqPropsObj.hasHttpPar("dump_object_info")):
# Dump status of all objects allocated into file specified.
targFile = reqPropsObj.getHttpPar("dump_object_info")
rmFile(targFile)
fo = open(targFile, "w")
fo.write(_genObjectStatus())
fo.close()
# Handle request for file info.
genCfgStatus = 0
genDiskStatus = 0
genFileStatus = 0
genStatesStatus = 1
genRequestStatus = 0
msg = ""
help = 0
if (reqPropsObj.hasHttpPar("help")):
global _help
msg = _help
help = 1
elif hostId and hostId != srvObj.getHostId():
host, port = srvObj.get_remote_server_endpoint(hostId)
cfgObj = srvObj.getCfg()
if not cfgObj.getProxyMode():
httpRef.redirect(host, port)
return
else:
try:
httpRef.proxy_request(hostId, host, port)
except Exception as e:
ex = re.sub("<|>", "", str(e))
errMsg = genLog("NGAMS_ER_COM",
[host, port,ex])
raise Exception(errMsg)
return
elif (fileList):
if (not fileListId):
# It's a new STATUS?file_list request.
fileListId = _handleFileList(srvObj, reqPropsObj, httpRef)
# Send back data from the request.
_handleFileListReply(srvObj, reqPropsObj, httpRef, fileListId,
maxElements)
elif (diskId):
diskObj = ngamsDiskInfo.ngamsDiskInfo()
diskObj.read(srvObj.getDb(), diskId)
status.addDiskStatus(diskObj)
genDiskStatus = 1
elif (fileId):
if (not fileVersion): fileVersion = -1
try:
fileObj = ngamsFileInfo.ngamsFileInfo()
fileObj.read(srvObj.getHostId(), srvObj.getDb(), fileId, fileVersion)
diskObj = ngamsDiskInfo.ngamsDiskInfo()
try:
diskObj.read(srvObj.getDb(), fileObj.getDiskId())
except:
errMsg = "Illegal Disk ID found: {0} for file with ID: {1}".format(fileObj.getDiskId(), fileId)
raise Exception(errMsg)
diskObj.addFileObj(fileObj)
status.addDiskStatus(diskObj)
except:
# The file was not found in the database. Check if it is available
# on a remote partner site.
host, port, status_info, disk_info, file_info = \
ngamsFileUtils.lookup_partner_site_file_status(srvObj, fileId,
fileVersion,
reqPropsObj)
# Update status reply using the disk status and file status
# information retrieved from the partner site
status.addDiskStatus(disk_info)
genDiskStatus = 1
genFileStatus = 1
elif (requestId):
logger.debug("Checking status of request with ID: %s", requestId)
reqPropsObjRef = srvObj.getRequest(requestId)
if (not reqPropsObjRef):
errMsg = genLog("NGAMS_ER_ILL_REQ_ID", [requestId])
raise Exception(errMsg)
genRequestStatus = 1
elif (configurationFile):
msg = "configuration_file=" + srvObj.cfg_fname
genCfgStatus = 1
# Hidden feature to return complete Cfg!!
try:
genCfgStatus = int(configurationFile)
except ValueError:
genCfgStatus = 1
status.setNgamsCfgObj(srvObj.getCfg())
elif (fileAccess):
if (not fileVersion): fileVersion = -1
fileId = fileAccess
msg = _checkFileAccess(srvObj, reqPropsObj, httpRef, fileId,
fileVersion, diskId)
elif (dbTime):
logger.debug("Querying total DB time")
msg = "Total DB time: %.6fs" % srvObj.getDb().getDbTime()
elif (dbTimeReset):
msg = "Resetting DB timer"
logger.debug(msg)
srvObj.getDb().resetDbTime()
else:
msg = "Successfully handled command STATUS"
if (reqPropsObjRef == reqPropsObj):
reqPropsObj.setCompletionTime()
srvObj.updateRequestDb(reqPropsObj)
if (genCfgStatus or genDiskStatus or genFileStatus or genRequestStatus):
status.setReqStatFromReqPropsObj(reqPropsObjRef)
# Generate XML reply.
logger.debug("Status requests: " + str([genCfgStatus, genDiskStatus, genFileStatus,
genStatesStatus]))
xmlStat = status.genXmlDoc(genCfgStatus, genDiskStatus, genFileStatus,
genStatesStatus)
xmlStat = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, xmlStat)
httpRef.send_data(six.b(xmlStat), NGAMS_XML_MT)
elif not httpRef.reply_sent:
httpRef.send_status(msg)
if (msg and (not help)):
logger.info(msg)
else:
logger.info("Successfully handled command STATUS")
def _registerExec(srvObj,
fileListDbmName,
tmpFilePat,
diskInfoDic,
reqPropsObj = None):
"""
Register the files listed in the File List DBM (ngamsDbm), which
match the mime-type(s) either specified in the 'mimeType' parameter,
or if this is not specified, which match all the mime-types specified
in the configuration file.
When the registration procedure has been executed, the function sends
an Email Notification message indicating which files were registered
if the HTTP parameter 'notif_email' is given.
The functions creates a File Info Objects per file handled and
writes this in a temporary file, which is a DBM file. The
keys in this DB is simply the file number in the sequence of files
handled, pointing to a pickled ngamsFileInfo object.
Each of the File Info Objects indicates if the file was registered or not.
This is done by setting the tag of the File Info Object to one of the
following values:
REGISTERED: The file was successfully registered in the NGAS DB
FAILED: The file was selected for registration but could not
be properly registered because of inconsistencies.
The status will be of the format: 'FAILED[: <reason>]'.
REJECTED: A file found under the specified path directory was
not accepted for cloning, usually because the mime-type
was not correct. The status will be of the format:
'REJECTED[: <reason>]'.
Note, that registration is comparable to archiving of files. For that
reason a DAPI must be provided for each type of file that should be
registered. If this is not fullfilled, the file registration will
fail.
Only files stored on one of the NGAS disks configured in the
configuration file, are considered. Files stored in other locations
are ignored.
srvObj: Instance of NG/AMS Server object (ngamsServer).
fileListDbmName: Name of a DBM containing the information
about the files to be registered. Each element in the
list is referred to by a key, which is a number.
These points to a pickled list for each file containing
the following information:
[<Filename>, <Disk ID>, <Mime-Type>]
The information for each disk concerned is also contained
in the DB referred to by its Disk ID and by its mount
point. The data is a pickled instance of the ngamsDiskInfo
class (string).
tmpFilePat: Pattern for temporary files used during the registration
process (string).
diskInfoDic: Dictionary with Disk IDs as keys pointing to the info
about the disk (dictionary/ngamsDiskInfo).
reqPropsObj: If an NG/AMS Request Properties Object is given, the
Request Status will be updated as the request is carried
out (ngamsReqProps).
Returns: Void.
"""
T = TRACE()
emailNotif = 0
if (reqPropsObj):
if (reqPropsObj.hasHttpPar("notif_email")):
emailNotif = 1
# Create the temporary BSD DB to contain the information for the
# Email Notification Message.
if (emailNotif):
regDbmName = tmpFilePat + "_NOTIF_EMAIL"
regDbm = ngamsDbm.ngamsDbm(regDbmName, writePerm = 1)
# Open the DBM containing the list of files to (possibly) register.
fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm = 1)
# Want to parse files in alphabetical order.
# TODO: Portatibility issue. Try to avoid UNIX shell commands for sorting.
tmpFileList = tmpFilePat + "_FILE_LIST"
rmFile(tmpFileList)
fo = open(tmpFileList, "w")
fileListDbm.initKeyPtr()
while (1):
dbmKey, fileInfo = fileListDbm.getNext()
if (not dbmKey): break
fo.write(dbmKey + "\n")
fo.close()
sortFileList = tmpFilePat + "_SORT_FILE_LIST"
rmFile(sortFileList)
shellCmd = "sort %s > %s" % (tmpFileList, sortFileList)
stat, out = commands.getstatusoutput(shellCmd)
if (stat != 0):
raise Exception, "Error executing command: %s. Error: %s" %\
(shellCmd, str(out))
rmFile(tmpFileList)
# Go through each file in the list, check if the mime-type is among the
# ones, which apply for registration. If yes try to register the file
# by invoking the corresponding DAPI on the file.
checksumPlugIn = srvObj.getCfg().getChecksumPlugIn().strip()
fileRegCount = 0
fileFailCount = 0
fileRejectCount = 0
regTimeAccu = 0.0
fileCount = 0
fo = open(sortFileList)
run = 1
while (run):
reg_start = time.time()
dbmKey = fo.readline()
if (dbmKey.strip() == ""):
run = 0
continue
fileInfo = fileListDbm.get(dbmKey[0:-1])
filename = fileInfo[0]
diskId = fileInfo[1]
mimeType = fileInfo[2]
# Register the file. Check first, that exactly this file is
# not already registered. In case it is, the file will be rejected.
regPi = srvObj.getCfg().register_plugins[mimeType]
logger.debug("Plugin found for %s: %s", mimeType, regPi)
params = ngamsPlugInApi.parseRawPlugInPars(regPi.pars)
tmpReqPropsObj = ngamsReqProps.ngamsReqProps().\
setMimeType(mimeType).\
setStagingFilename(filename).\
setTargDiskInfo(diskInfoDic[diskId]).\
setHttpMethod(NGAMS_HTTP_GET).\
setCmd(NGAMS_REGISTER_CMD).\
setSize(os.path.getsize(filename)).\
setFileUri(filename).\
setNoReplication(1)
tmpFileObj = ngamsFileInfo.ngamsFileInfo()
try:
# Invoke Registration Plug-In.
piName = regPi.name
plugInMethod = loadPlugInEntryPoint(piName)
piRes = plugInMethod(srvObj, tmpReqPropsObj, params)
del tmpReqPropsObj
# Check if this file is already registered on this disk. In case
# yes, it is not registered again.
files = srvObj.db.getFileSummary1(srvObj.getHostId(), [piRes.getDiskId()],
[piRes.getFileId()])
fileRegistered = 0
for tmpFileInfo in files:
tmpMtPt = tmpFileInfo[ngamsDbCore.SUM1_MT_PT]
tmpFilename = tmpFileInfo[ngamsDbCore.SUM1_FILENAME]
tmpComplFilename = os.path.normpath(tmpMtPt + "/" +\
tmpFilename)
if (tmpComplFilename == filename):
fileRegistered = 1
break
if (fileRegistered):
fileRejectCount += 1
tmpMsgForm = "REJECTED: File with File ID/Version: %s/%d " +\
"and path: %s is already registered on disk " +\
"with Disk ID: %s"
tmpMsg = tmpMsgForm % (piRes.getFileId(),
piRes.getFileVersion(), filename,
piRes.getDiskId())
logger.warning(tmpMsg + ". File is not registered again.")
if (emailNotif):
tmpFileObj.\
setDiskId(diskId).setFilename(filename).\
setTag(tmpMsg)
regDbm.addIncKey(tmpFileObj)
if (reqPropsObj):
reqPropsObj.incActualCount(1)
ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj,
regTimeAccu)
regTimeAccu += time.time() - reg_start
fileCount += 1
continue
# Calculate checksum (if plug-in specified).
if (checksumPlugIn != ""):
logger.debug("Invoking Checksum Plug-In: %s to handle file: %s",
checksumPlugIn, filename)
plugInMethod = loadPlugInEntryPoint(checksumPlugIn)
checksum = plugInMethod(srvObj, filename, 0)
else:
checksum = ""
# Move file and update information about file in the NGAS DB.
mvFile(filename, piRes.getCompleteFilename())
ngamsArchiveUtils.updateFileInfoDb(srvObj, piRes, checksum,
checksumPlugIn)
ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), piRes)
ngamsLib.makeFileReadOnly(piRes.getCompleteFilename())
if (emailNotif):
uncomprSize = piRes.getUncomprSize()
ingestDate = time.time()
creDateSecs = getFileCreationTime(filename)
tmpFileObj.\
setDiskId(diskId).\
setFilename(filename).\
setFileId(piRes.getFileId()).\
setFileVersion(piRes.getFileVersion()).\
setFormat(piRes.getFormat()).\
setFileSize(piRes.getFileSize()).\
setUncompressedFileSize(uncomprSize).\
setCompression(piRes.getCompression()).\
setIngestionDate(ingestDate).\
setIgnore(0).\
setChecksum(checksum).\
setChecksumPlugIn(checksumPlugIn).\
setFileStatus(NGAMS_FILE_STATUS_OK).\
setCreationDate(creDateSecs).\
setTag("REGISTERED")
fileRegCount += 1
# If running as a cache archive, update the Cache New Files DBM
# with the information about the new file.
if (srvObj.getCachingActive()):
fileVer = fio.getFileVersion()
ngamsCacheControlThread.addEntryNewFilesDbm(srvObj, diskId,
piRes.getFileId(),
fileVer, filename)
# Generate a confirmation log entry.
msg = genLog("NGAMS_INFO_FILE_REGISTERED",
[filename, piRes.getFileId(), piRes.getFileVersion(),
piRes.getFormat()])
time.sleep(0.005)
regTime = time.time() - reg_start
msg = msg + ". Time: %.3fs." % (regTime)
logger.info(msg, extra={'to_syslog': 1})
except Exception, e:
errMsg = genLog("NGAMS_ER_FILE_REG_FAILED", [filename, str(e)])
logger.error(errMsg)
if (emailNotif):
tmpFileObj.\
setDiskId(diskId).setFilename(filename).\
setTag(errMsg)
fileFailCount += 1
regTime = time.time() - reg_start
# TODO (rtobar, 2016-01): Why don't we raise an exception here?
# Otherwise the command appears as successful on the
# client-side
# Add the file information in the registration report.
if (emailNotif): regDbm.addIncKey(tmpFileObj)
# Update request status time information.
regTimeAccu += regTime
if (reqPropsObj):
reqPropsObj.incActualCount(1)
ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj,
regTimeAccu)
fileCount += 1
def receiveData(srvObj,
reqPropsObj,
httpRef):
"""
Receive the data in connection with the Rearchive Request.
For a description of the parameters: Check handleCmd().
Returns: Tuple with File Info Object for the file to be rearchived and
Disk Info Object for the selected target disk
(tuple/(fileInfo, ngamsDiskInfo)).
"""
# Note, this algorithm does not implement support for back-log buffering
# for speed optimization reasons.
# Unpack the file information contained in the HTTP header
# (NGAS-File-Info).
encFileInfo = reqPropsObj.getHttpHdr(NGAMS_HTTP_HDR_FILE_INFO)
if (not encFileInfo):
msg = "Error. Must provide NGAS File Information " +\
"(RFC 3548 encoded) in HTTP header NGAS-File-Info"
raise Exception(msg)
fileInfoXml = base64.b64decode(encFileInfo)
fileInfoObj = ngamsFileInfo.ngamsFileInfo().unpackXmlDoc(fileInfoXml)
# Find the most suitable target Disk Set.
trgDiskInfoObj = None
diskExList = []
while (True):
try:
tmpTrgDiskInfo = ngamsDiskUtils.\
findTargetDisk(srvObj.getHostId(),
srvObj.getDb(), srvObj.getCfg(),
reqPropsObj.getMimeType(),
sendNotification = 0,
diskExemptList = diskExList,
reqSpace = reqPropsObj.getSize())
if (srvObj.getDb().fileInDb(tmpTrgDiskInfo.getDiskId(),
fileInfoObj.getFileId(),
fileInfoObj.getFileVersion())):
diskExList.append(tmpTrgDiskInfo.getDiskId())
else:
trgDiskInfoObj = tmpTrgDiskInfo
break
except Exception as e:
msg = "Error locating target disk for REARCHIVE Command. Error: %s"
msg = msg % str(e)
raise Exception(msg)
# Generate Staging Filename + save file into this.
storageSetId = trgDiskInfoObj.getStorageSetId()
stagingFilename = ngamsHighLevelLib.\
genStagingFilename(srvObj.getCfg(), reqPropsObj,
srvObj.getDiskDic(), storageSetId,
reqPropsObj.getFileUri(),
genTmpFiles = 0)
reqPropsObj.setStagingFilename(stagingFilename)
# If it is an Rearchive Pull Request, open the URL.
if (reqPropsObj.getHttpMethod() == NGAMS_HTTP_GET):
# urllib.urlopen will attempt to get the content-length based on the URI
# i.e. file, ftp, http
handle = urllib.urlopen(reqPropsObj.getFileUri())
reqPropsObj.setSize(handle.info()['Content-Length'])
rfile = handle
else:
reqPropsObj.setSize(fileInfoObj.getFileSize())
rfile = httpRef.rfile
# Save the data into the Staging File.
try:
# Make mutual exclusion on disk access (if requested).
ngamsHighLevelLib.acquireDiskResource(srvObj.getCfg(), trgDiskInfoObj.getSlotId())
# If provided with a checksum, calculate the checksum on the incoming
# data stream and check that it is the same as the expected one
# (as indicated in the file info structure given by the user)
stored_checksum = fileInfoObj.getChecksum()
crc_variant = fileInfoObj.getChecksumPlugIn()
skip_crc = True
if stored_checksum and crc_variant:
reqPropsObj.addHttpPar('crc_variant', crc_variant)
reqPropsObj.__httpHdrDic[NGAMS_HTTP_HDR_CHECKSUM] = stored_checksum
skip_crc = False
ngamsArchiveUtils.archive_contents_from_request(stagingFilename, srvObj.getCfg(),
reqPropsObj, rfile, skip_crc=skip_crc)
finally:
ngamsHighLevelLib.releaseDiskResource(srvObj.getCfg(), trgDiskInfoObj.getSlotId())
# Synchronize the file caches to ensure the files have been stored
# on the disk and check that the files are accessible.
# This sync is only relevant if back-log buffering is on.
if (srvObj.getCfg().getBackLogBuffering()):
ngamsFileUtils.syncCachesCheckFiles(srvObj, [stagingFilename])
return (fileInfoObj, trgDiskInfoObj)
return
elif (fileList):
if (not fileListId):
# It's a new STATUS?file_list request.
fileListId = _handleFileList(srvObj, reqPropsObj, httpRef)
# Send back data from the request.
_handleFileListReply(srvObj, reqPropsObj, httpRef, fileListId,
maxElements)
elif (diskId):
diskObj = ngamsDiskInfo.ngamsDiskInfo()
diskObj.read(srvObj.getDb(), diskId)
status.addDiskStatus(diskObj)
genDiskStatus = 1
elif (fileId):
if (not fileVersion): fileVersion = -1
fileObj = ngamsFileInfo.ngamsFileInfo()
fileObj.read(srvObj.getHostId(), srvObj.getDb(), fileId, fileVersion)
diskObj = ngamsDiskInfo.ngamsDiskInfo()
try:
diskObj.read(srvObj.getDb(), fileObj.getDiskId())
except:
errMsg = "Illegal Disk ID found: %s for file with ID: %s" %\
(fileObj.getDiskId(), fileId)
raise Exception, errMsg
diskObj.addFileObj(fileObj)
status.addDiskStatus(diskObj)
genDiskStatus = 1
genFileStatus = 1
elif (requestId):
logger.debug("Checking status of request with ID: %s", requestId)
reqPropsObjRef = srvObj.getRequest(requestId)
def _handleFileListReply(srvObj,
reqPropsObj,
httpRef,
fileListId,
maxElements = None):
"""
Extracts file information from a previously dumped file information
in connection with a STATUS?file_list request.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler object
(ngamsHttpRequestHandler).
fileListId: File List ID allocated to this request (string).
maxElements: Maximum number of elements to extract and return to the
requestor (integer).
Returns: Void.
"""
# Get the name of the DBM in which the information is stored.
dbmBaseName = STATUS_FILE_LIST_DBM_TAG % fileListId
dbmPat = os.path.normpath("%s/*%s*" %\
(ngamsHighLevelLib.\
getNgasTmpDir(srvObj.getCfg()),
dbmBaseName))
dbmMatches = glob.glob(dbmPat)
if (len(dbmMatches) < 1):
msg = "Referenced File List ID: %s in connection with " +\
"STATUS/file_list request, is not (or no longer) known"
raise Exception(msg % fileListId)
elif (len(dbmMatches) > 1):
msg = "Inconsistencies encountered in locating result set for " +\
"STATUS/file_list for referenced File List ID: %s"
raise Exception(msg % fileListId)
fileInfoDbmName = dbmMatches[0]
# Generate the NG/AMS Status Document, with a File List in it.
# Compress it on the fly.
fileListXmlDoc = ngamsHighLevelLib.\
genTmpFilename(srvObj.getCfg(),
"STATUS_FILE_LIST_XML.xml")
try:
tmpFileListDbm = ngamsDbm.ngamsDbm(fileInfoDbmName,
cleanUpOnDestr = 0,
writePerm = 1)
rmFile(fileListXmlDoc)
fo = open(fileListXmlDoc, "w")
if (not maxElements):
remainingObjects = 0
else:
remainingObjects = (tmpFileListDbm.getCount() - maxElements)
if (remainingObjects < 0):
remainingObjects = 0
fo.write(_fileListXmlHdr % (fileListId, str(remainingObjects)))
# Loop over the file info objects and write them into the file.
# take max the number of elements specified.
tmpFileListDbm.initKeyPtr()
elCount = 0
keyRefList = []
while (True):
# Have the requested number of elements been extracted?
if (maxElements):
if (elCount >= maxElements):
break
# Get the next key (if there are more elements).
key, fileInfo = tmpFileListDbm.getNext()
if (not key):
break
try:
# Write the File Status XML Element in the file.
tmpFileInfoObj = ngamsFileInfo.ngamsFileInfo().\
unpackSqlResult(fileInfo)
fileInfoXml = tmpFileInfoObj.genXml(storeDiskId = 1).\
toprettyxml(" ", "\n")[:-1]
fo.write("\n" + fileInfoXml)
except Exception as e:
msg = "Error creating STATUS/File List XML Document. " +\
"Error: %s" % str(e)
logger.error(msg)
raise
keyRefList.append(key)
elCount += 1
# Finish up the XML document, close the file.
fo.write(_fileListXmlFooter)
fo.close()
# Assume this type of file can always be compressed.
fileListXmlDoc = compressFile(fileListXmlDoc)
except:
rmFile("%s*" % fileInfoDbmName)
rmFile("%s*" % fileListXmlDoc)
raise
# Send the XML document back to the requestor.
try:
httpRef.send_file(fileListXmlDoc, NGAMS_GZIP_XML_MT)
# Remove the reported entries.
for key in keyRefList:
tmpFileListDbm.rem(key)
tmpFileListDbm.sync()
del keyRefList
keyRefList = []
# If there are no more entries, delete the DBM.
dbmCount = tmpFileListDbm.getCount()
del tmpFileListDbm
if (dbmCount == 0):
rmFile("%s*" % fileInfoDbmName)
except Exception as e:
msg = "Error returning response to STATUS?file_list request. Error: %s"
msg = msg % str(e)
raise Exception(msg)
finally:
rmFile(fileListXmlDoc)
fileInfoList[ngamsDb.NGAS_FILES_CHECKSUM_PI] = checksumPlugIn
fileInfoList[ngamsDb.NGAS_FILES_FILE_STATUS] = "00000000"
fileInfoList[ngamsDb.NGAS_FILES_CREATION_DATE] = ingestionTime
updateDbSnapshot(dbSnapshotDbm, fileInfoList)
del fileInfoList
# Update DB if requested.
if (dbCon):
fileInfoObj = ngamsFileInfo.ngamsFileInfo().\
setChecksum(checksum).\
setChecksumPlugIn(checksumPlugIn).\
setCompression(compression).\
setCreationDate(ingestionTime).\
setDiskId(diskId).\
setFileId(fileId).\
setFileSize(archFileSize).\
setFileStatus("00000000").\
setFileVersion(fileVer).\
setFilename(relFilename).\
setFormat(format).\
setIgnore(0).\
setIngestionDate(ingestionTime).\
setUncompressedFileSize(uncomprSize)
fileInfoObj.write(getHostName(), dbCon, genSnapshot=0)
del fileInfoObj
dayFileCount += 1
fileCount += 1
# Give a signal of life ...
if ((fileCount % 100) == 0):
def checkFileCopiesAndReg(srvObj,
minReqCopies,
dbFilePat,
fileListDbmName=None,
diskId=None,
ignoreMounted=0):
"""
The function checks for each file referenced if there are at least
'minReqCopies' copies available somewhere in this NGAS cluster. For the
files where this is not the case, an entry is added in a ngasDiskInfo
object indicating that this file
If an entire disk is analysed (about to be deleted), it is also checked if
each file stored on the disk is registered in the DB. Otherwise an NG/AMS
File List is returned, containing references to the files not registered.
srvObj: Instance of the NG/AMS Server Class (ngamsServer).
minReqCopies: Minimum number of copies required (integer).
dbFilePat: Filename pattern used to build the DBM containing
information about the files (string).
fileListDbmName: Name of DBM DB containing explicit references to files
to be checked if they can be deleted. The information
in this table is pickled lists with the lay-out defined
by ngamsDb._ngasFilesCols (string).
diskId: Used to refer to all files stored on a disk (string|None).
ignoreMounted: Carry out the check also if the disk is not mounted
(integer/0|1).
Returns: Tuple contaning the filenames of three DBM DBs with the
following information:
o Files not having the specified number of copies.
The contents of this DB are keys (Disk ID + File ID +
File Version), pointing to pickled ngamsFileInfo
objects.
o Files found on the disk but not registered.
This contains the complete filenames of files found
on the disk, which are not registered. These filenames
are the keys of this DBM DB.
o Complete list of files referenced in connection
with the query. The contents of this DB are keys,
which are a simple counter pointing to pickled
list containing the information as returned by
ngamsDb.getFileSummary1().
(tuple/string).
"""
T = TRACE()
if ((not fileListDbmName) and (not diskId)):
errMsg = "ngamsSrvUtils.checkFileCopiesAndReg(): Must specify " +\
"either a DBM with files to be checked or a Disk ID"
logger.warning(errMsg)
raise Exception(errMsg)
# Create DBMs:
# - DB containing information about files having less then the
# specified number of copies.
fileMisCopyDbmName = os.path.normpath(dbFilePat + "_MISSING_COPIES")
fileMisCopyDbm = ngamsDbm.ngamsDbm(fileMisCopyDbmName, writePerm=1)
# - DB that contains information about files stored on the DB,
# which are not registered in the NGAS DB. At the end of the function,
# this will contain information about files found on the disk but
# not registered in the NGAS DB.
filesOnDiskDicDbmName = os.path.normpath(dbFilePat + "_FILES_ON_DISK")
filesOnDiskDicDbm = ngamsDbm.ngamsDbm(filesOnDiskDicDbmName, writePerm=1)
# - DB with information about files referenced by the query.
if (not fileListDbmName):
locFileListDbmName = os.path.normpath(dbFilePat + "_FILE_LIST")
fileListDbm = ngamsDbm.ngamsDbm(locFileListDbmName, writePerm=1)
else:
fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm=0)
# - Temporary DBM containing information about all File IDs defined
# by the query.
fileIdDbmName = os.path.normpath(dbFilePat + "_FILE_IDS")
fileIdDbm = ngamsDbm.ngamsDbm(fileIdDbmName, writePerm=1)
# - Temporary DBM containing information about all files available in
# the system with the File ID/File Version defined by the query.
complFileListDbmName = os.path.normpath(dbFilePat + "_COMPL_FILE_LIST")
complFileListDbm = ngamsDbm.ngamsDbm(complFileListDbmName, writePerm=1)
# - Temporary DBM that is used to figure out the number of independent
# copies of each file concerned by the query.
checkDicDbmName = os.path.normpath(dbFilePat + "_CHECK_DIC")
checkDicDbm = ngamsDbm.ngamsDbm(checkDicDbmName, writePerm=1)
# A Disk ID but no file references are given. Retrieve information
# about files concerned from the DB.
if (diskId):
logger.debug("Retrieving information about files on disk with ID: %s",
diskId)
for f in srvObj.db.getFileSummary1(None, [diskId], [],
ignore=0,
fileStatus=[]):
fileListDbm.addIncKey(f)
fileId = f[ngamsDbCore.SUM1_FILE_ID]
fileIdDbm.add(str(fileId), "")
fileVersion = f[ngamsDbCore.SUM1_VERSION]
fileKey = ngamsLib.genFileKey(None, fileId, fileVersion)
checkDicDbm.add(fileKey, {})
fileListDbm.sync()
fileIdDbm.sync()
checkDicDbm.sync()
# Get the list of files located on the disk. Later on, remove entries
# from this dictionary as the files are parsed, based on their DB info,
# further down in this method.
#
# Key in this dictionary is the complete filename of the file.
logger.debug("Get list of files stored on disk ...")
tmpDiskInfo = srvObj.getDb().getDiskInfoFromDiskId(diskId)
diskInfoObj = ngamsDiskInfo.ngamsDiskInfo().\
unpackSqlResult(tmpDiskInfo)
if ((not ignoreMounted) and (not diskInfoObj.getMounted())):
errMsg = "Rejecting request for removing disk with ID: " +\
diskId + " - disk not mounted!"
raise Exception(errMsg)
if (not ignoreMounted):
basePath = os.path.normpath(diskInfoObj.getMountPoint())
pattern = "/*"
logger.debug(
"Generating list with files on disk with base path: %s",
basePath)
while (1):
tmpFileList = glob.glob(basePath + pattern)
if (len(tmpFileList) == 0):
break
else:
for filename in tmpFileList:
if (os.path.isfile(filename) and
(os.path.basename(filename) != NGAMS_DISK_INFO)
and (os.path.basename(filename) !=
NGAMS_VOLUME_ID_FILE)
and (os.path.basename(filename) !=
NGAMS_VOLUME_INFO_FILE)):
filesOnDiskDicDbm.add(str(filename), "")
pattern += "/*"
# Generate File ID DBM in case a file list DBM is given.
if (fileListDbmName):
logger.debug("Handling file list DBM given in the function call ...")
fileListDbm.initKeyPtr()
while (1):
key, tmpFileInfo = fileListDbm.getNext()
if (not key): break
# Update the File ID DBM.
fileId = tmpFileInfo[ngamsDbCore.SUM1_FILE_ID]
fileIdDbm.add(str(fileId), "")
# Update the DBM with references to File ID/Version sets.
fileVersion = tmpFileInfo[ngamsDbCore.SUM1_VERSION]
fileKey = ngamsLib.genFileKey(None, fileId, fileVersion)
checkDicDbm.add(fileKey, {})
fileIdDbm.sync()
checkDicDbm.sync()
# We need to generate a list with all files available in the system
# with the given File ID/File Version.
logger.debug("Retrieving information about all files available with the " +\
"File ID/File Version as defined by the query")
# Due to the limitation of the size of SQL queries, we have to split up
# the SQL query in several sub-queries. The max. length of an SQL query
# is defined by NGAMS_MAX_SQL_QUERY_SZ, we subtract 512 from this for
# the general part of the query, and for each filename we calculate a
# length of len(File ID) + 4 as contribution to the SQL query.
maxQuerySize = (NGAMS_MAX_SQL_QUERY_SZ - 512)
queryIds = []
querySize = 0
noOfFileIds = fileIdDbm.getCount()
fileIdCount = 0
fileIdDbm.initKeyPtr()
fileId = "INIT"
while (fileId):
fileId, dummy = fileIdDbm.getNext()
if (fileId):
queryIds.append(utils.b2s(fileId))
fileIdCount += 1
querySize += (len(fileId) + 4)
if queryIds and (querySize >= maxQuerySize
or fileIdCount == noOfFileIds):
for f in srvObj.db.getFileSummary1(fileIds=queryIds,
fileStatus=[]):
# Take only a sub-result if that File ID + Version
# is concerned by the query.
tmpFileId = f[ngamsDbCore.SUM1_FILE_ID]
tmpFileVersion = f[ngamsDbCore.SUM1_VERSION]
tmpFileKey = ngamsLib.genFileKey(None, tmpFileId,
tmpFileVersion)
if (checkDicDbm.hasKey(tmpFileKey)):
complFileListDbm.addIncKey(f)
complFileListDbm.sync()
queryIds = []
querySize = 0
# Now, go through the files found and order these such that we end up with
# a Dictionary with "<File ID>_<File Version>" as keys referring
# to a dictionary with the Disk IDs of the disks hosting the files as
# keys, and the information for each file on that disk as a tupple.
#
# It is the intention to figure out how many copies we have of each file
# identified by File ID + File Version stored ON DIFFERENT STORAGE MEDIAS
# + on different hosts.
logger.debug("Generate DBM DB with info about independent file copies ...")
complFileListDbm.initKeyPtr()
while (1):
fileKey, fileInfo = complFileListDbm.getNext()
if (not fileKey): break
checkDicKey = ngamsLib.genFileKey(None,
fileInfo[ngamsDbCore.SUM1_FILE_ID],
fileInfo[ngamsDbCore.SUM1_VERSION])
tmpDic = checkDicDbm.get(checkDicKey)
tmpDic[fileInfo[ngamsDbCore.SUM1_DISK_ID]] = fileInfo
checkDicDbm.add(checkDicKey, tmpDic)
# Check if there are at least minReqCopies occurrences of the files +
# check that all files are registered (if a Disk ID is specified).
logger.debug("Check for files with less copies than: %s",
str(minReqCopies))
checkDicDbm.initKeyPtr()
while (1):
checkDicKey, tmpDic = checkDicDbm.getNext()
if (not checkDicKey): break
tmpDicKeys = list(tmpDic)
noOfCopies = len(tmpDicKeys)
if (noOfCopies < minReqCopies):
tmpFileInfo = tmpDic[tmpDicKeys[0]]
fileId = tmpFileInfo[ngamsDbCore.SUM1_FILE_ID]
fileVersion = tmpFileInfo[ngamsDbCore.SUM1_VERSION]
tmpFileObj = ngamsFileInfo.ngamsFileInfo().\
setFileId(fileId).\
setFileVersion(fileVersion).\
setTag("Independent copies: " + str(noOfCopies))
fileKey = ngamsLib.genFileKey(None, fileId, fileVersion)
fileMisCopyDbm.add(str(fileKey), tmpFileObj)
# Remove this file from the Files On Disk DBM - do this only
# if a Disk ID is specified.
if (diskId):
if (diskId in tmpDic):
fileInfo = tmpDic[diskId]
filename = os.path.\
normpath(fileInfo[ngamsDbCore.SUM1_MT_PT] +\
"/" + fileInfo[ngamsDbCore.SUM1_FILENAME])
filename = str(filename)
if (filesOnDiskDicDbm.hasKey(filename)):
filesOnDiskDicDbm.rem(filename)
# Close all DBM objects.
del fileMisCopyDbm
del filesOnDiskDicDbm
del fileListDbm
del fileIdDbm
del complFileListDbm
del checkDicDbm
# The DBM filesOnDiskDicDbmName now contains references to files,
# which are found on the disk but not registered in the DB.
return (fileMisCopyDbmName, filesOnDiskDicDbmName, complFileListDbmName)
def handleCmd(srvObj, reqPropsObj, httpRef):
"""
Handle the CARCHIVE Command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: (fileId, filePath) tuple.
"""
T = TRACE()
# Is this NG/AMS permitted to handle Archive Requests?
logger.debug("Is this NG/AMS permitted to handle Archive Requests?")
if (not srvObj.getCfg().getAllowArchiveReq()):
errMsg = genLog("NGAMS_ER_ILL_REQ", ["Archive"])
raise Exception(errMsg)
srvObj.checkSetState("Archive Request", [NGAMS_ONLINE_STATE],
[NGAMS_IDLE_SUBSTATE, NGAMS_BUSY_SUBSTATE],
NGAMS_ONLINE_STATE,
NGAMS_BUSY_SUBSTATE,
updateDb=False)
if httpRef.command != 'POST':
raise Exception("Only POST allowed for CARCHIVE")
# Get mime-type (try to guess if not provided as an HTTP parameter).
logger.debug(
"Get mime-type (try to guess if not provided as an HTTP parameter).")
if (reqPropsObj.getMimeType() == ""):
mimeType = ngamsHighLevelLib.\
determineMimeType(srvObj.getCfg(), reqPropsObj.getFileUri())
reqPropsObj.setMimeType(mimeType)
else:
mimeType = reqPropsObj.getMimeType()
# Determine the target volume, ignoring the stream concept.
logger.debug("Determine the target volume, ignoring the stream concept.")
targDiskInfo = ngamsArchiveUtils._random_target_volume(srvObj)
if (targDiskInfo == None):
errMsg = "No disk volumes are available for ingesting any files."
raise Exception(errMsg)
reqPropsObj.setTargDiskInfo(targDiskInfo)
# Generate staging filename.
logger.debug("Generate staging filename from URI: %s",
reqPropsObj.getFileUri())
if (reqPropsObj.getFileUri().find("file_id=") >= 0):
file_id = reqPropsObj.getFileUri().split("file_id=")[1]
baseName = os.path.basename(file_id)
else:
baseName = os.path.basename(reqPropsObj.getFileUri())
stgFilename = os.path.join("/", targDiskInfo.getMountPoint(),
NGAMS_STAGING_DIR,
genUniqueId() + "___" + baseName)
logger.debug("Staging filename is: %s", stgFilename)
reqPropsObj.setStagingFilename(stgFilename)
# Retrieve file contents (from URL, archive pull, or by storing the body
# of the HTTP request, archive push).
stagingInfo = saveInStagingFile(srvObj.getCfg(), reqPropsObj, httpRef,
stgFilename, targDiskInfo)
ioTime = stagingInfo[0]
rootContainer = stagingInfo[1]
fileDataList = stagingInfo[2]
ingestRate = stagingInfo[3]
reqPropsObj.incIoTime(ioTime)
createContainers(rootContainer, None, srvObj)
parDic = {}
ngamsGenDapi.handlePars(reqPropsObj, parDic)
diskInfo = reqPropsObj.getTargDiskInfo()
# Generate file information.
logger.debug("Generate file information")
dateDir = toiso8601(fmt=FMT_DATE_ONLY)
resDapiList = []
containerSizes = {}
for item in fileDataList:
container = item[0]
filepath = item[1]
crc = item[2]
containerId = str(container.getContainerId())
basename = os.path.basename(filepath)
fileId = basename
fileVersion, relPath, relFilename,\
complFilename, fileExists =\
ngamsPlugInApi.genFileInfo(srvObj.getDb(),
srvObj.getCfg(),
reqPropsObj, diskInfo,
filepath,
fileId,
basename, [dateDir])
complFilename, relFilename = ngamsGenDapi.checkForDblExt(
complFilename, relFilename)
# Keep track of the total size of the container
uncomprSize = ngamsPlugInApi.getFileSize(filepath)
if containerId not in containerSizes:
containerSizes[containerId] = 0
containerSizes[containerId] += uncomprSize
mimeType = reqPropsObj.getMimeType()
compression = "NONE"
archFileSize = ngamsPlugInApi.getFileSize(filepath)
resDapi = ngamsPlugInApi.genDapiSuccessStat(
diskInfo.getDiskId(), relFilename, fileId, fileVersion, mimeType,
archFileSize, uncomprSize, compression, relPath,
diskInfo.getSlotId(), fileExists, complFilename)
# Move file to final destination.
logger.debug("Moving file to final destination")
ioTime = mvFile(filepath, resDapi.getCompleteFilename())
reqPropsObj.incIoTime(ioTime)
# Get crc info
checksumPlugIn = "StreamCrc32"
checksum = str(crc)
# Get source file version
# e.g.: http://ngas03.hq.eso.org:7778/RETRIEVE?file_version=1&file_id=X90/X962a4/X1
logger.debug("Get file version")
file_version = resDapi.getFileVersion()
if reqPropsObj.getFileUri().count("file_version"):
file_version = int(
(reqPropsObj.getFileUri().split("file_version=")[1]
).split("&")[0])
# Check/generate remaining file info + update in DB.
logger.debug("Creating db entry")
creDate = getFileCreationTime(resDapi.getCompleteFilename())
fileInfo = ngamsFileInfo.ngamsFileInfo().\
setDiskId(resDapi.getDiskId()).\
setFilename(resDapi.getRelFilename()).\
setFileId(resDapi.getFileId()).\
setFileVersion(file_version).\
setFormat(resDapi.getFormat()).\
setFileSize(resDapi.getFileSize()).\
setUncompressedFileSize(resDapi.getUncomprSize()).\
setCompression(resDapi.getCompression()).\
setIngestionDate(time.time()).\
setChecksum(checksum).setChecksumPlugIn(checksumPlugIn).\
setFileStatus(NGAMS_FILE_STATUS_OK).\
setCreationDate(creDate).\
setIoTime(reqPropsObj.getIoTime())
fileInfo.write(srvObj.getHostId(), srvObj.getDb())
# Add the file to the container
srvObj.getDb().addFileToContainer(containerId, resDapi.getFileId(),
True)
# Update the container sizes
for contSizeInfo in containerSizes.items():
srvObj.getDb().setContainerSize(contSizeInfo[0], contSizeInfo[1])
# Inform the caching service about the new file.
logger.debug("Inform the caching service about the new file.")
if (srvObj.getCachingActive()):
diskId = resDapi.getDiskId()
fileId = resDapi.getFileId()
fileVersion = file_version
filename = resDapi.getRelFilename()
ngamsCacheControlThread.addEntryNewFilesDbm(
srvObj, diskId, fileId, fileVersion, filename)
# Update disk info in NGAS Disks.
logger.debug("Update disk info in NGAS Disks.")
srvObj.getDb().updateDiskInfo(resDapi.getFileSize(),
resDapi.getDiskId())
resDapiList.append(resDapi)
# Check if the disk is completed.
# We use an approximate extimate for the remaning disk space to avoid
# to read the DB.
logger.debug("Check available space in disk")
availSpace = getDiskSpaceAvail(targDiskInfo.getMountPoint(), smart=False)
if (availSpace < srvObj.getCfg().getFreeSpaceDiskChangeMb()):
targDiskInfo.setCompleted(1).setCompletionDate(time.time())
targDiskInfo.write(srvObj.getDb())
# Request after-math ...
srvObj.setSubState(NGAMS_IDLE_SUBSTATE)
msg = "Successfully handled Archive Pull Request for data file " +\
"with URI: " + reqPropsObj.getSafeFileUri()
logger.info(msg)
httpRef.send_ingest_status(msg, targDiskInfo)
for resDapi in resDapiList:
# Trigger Subscription Thread. This is a special version for MWA, in which we simply swapped MIRRARCHIVE and QARCHIVE
# [email protected]
logger.debug("triggering SubscriptionThread for file %s",
resDapi.getFileId())
srvObj.addSubscriptionInfo(
[(resDapi.getFileId(), resDapi.getFileVersion())], [])
srvObj.triggerSubscriptionThread()
