def processRequest(srvObj,
reqPropsObj,
httpRef,
fileInfoObj,
trgDiskInfoObj):
"""
Process the Rearchive Request.
For a description of the parameters: Check handleCmd().
fileInfoObj: File information for file to be restored (ngamsFileInfo).
trgDiskInfoObj: Disk Info Object for target disk (ngamsDiskInfo).
Returns: Void.
"""
# Generate the DB File Information.
newFileInfoObj = fileInfoObj.clone().\
setDiskId(trgDiskInfoObj.getDiskId()).\
setCreationDate(time.time())
# Generate the final storage location and move the file there.
targetFilename = os.path.normpath("%s/%s" %\
(trgDiskInfoObj.getMountPoint(),
newFileInfoObj.getFilename()))
logger.debug("Move Restore Staging File to final destination: %s->%s ...",
reqPropsObj.getStagingFilename(), targetFilename)
io_start = time.time()
mvFile(reqPropsObj.getStagingFilename(), targetFilename)
ioTime = time.time() - io_start
reqPropsObj.incIoTime(ioTime)
logger.debug("Moved Restore Staging File to final destination: %s->%s",
reqPropsObj.getStagingFilename(), targetFilename)
# Update the DB with the information about the new file.
# Update information for Main File/Disk in DB.
newFileInfoObj.write(srvObj.getHostId(), srvObj.getDb())
diskSpace = getDiskSpaceAvail(trgDiskInfoObj.getMountPoint())
newSize = (trgDiskInfoObj.getBytesStored() + newFileInfoObj.getFileSize())
ioTime = (trgDiskInfoObj.getTotalDiskWriteTime() + reqPropsObj.getIoTime())
trgDiskInfoObj.\
setNumberOfFiles(trgDiskInfoObj.getNumberOfFiles() + 1).\
setAvailableMb(diskSpace).setBytesStored(newSize).\
setTotalDiskWriteTime(ioTime).write(srvObj.getDb())
# Ensure that the restored file is readonly.
ngamsLib.makeFileReadOnly(targetFilename)
# Add the file info object for the target file in the disk info object.
trgDiskInfoObj.addFileObj(newFileInfoObj)
def handleCmd(srvObj, reqPropsObj, httpRef):
"""
Handle the Quick Archive (QARCHIVE) Command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: (fileId, filePath) tuple.
"""
# LARCHIVE was designed to work with the fileUri parameter, unlike the rest
# of the archiving commands that work with a combination of the Content-Disposition
# header and the filename parameter
reqPropsObj.setFileUri(reqPropsObj.get('fileUri', ''))
mimeType = ngamsArchiveUtils.archiveInitHandling(srvObj,
reqPropsObj,
httpRef,
do_probe=False,
try_to_proxy=False)
resDapi, targDiskInfo = ngamsArchiveUtils.archiveFromFile(
srvObj, reqPropsObj.getFileUri(), 0, mimeType, reqPropsObj)
if (resDapi == NGAMS_FAILURE):
errMsg = targDiskInfo
httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=500)
return
# Check if the disk is completed.
# We use an approximate estimate for the remaning disk space to avoid
# to read the DB.
logger.debug("Check available space in disk")
availSpace = getDiskSpaceAvail(targDiskInfo.getMountPoint(), smart=False)
if (availSpace < srvObj.getCfg().getFreeSpaceDiskChangeMb()):
targDiskInfo.setCompleted(1).setCompletionDate(time.time())
targDiskInfo.write(srvObj.getDb())
ngamsArchiveUtils.finish_archive_request(srvObj, reqPropsObj, httpRef,
resDapi, targDiskInfo)
def cloneCheckDiskSpace(srvObj, cloneListDbmName, tmpFilePat, targetDiskId=""):
"""
Check if there is enough disk space available on this NGAS host for
carrying out the Clone Request.
srvObj: Reference to instance of Server Object (ngamsServer).
cloneListDbmName: Name of DBM containing the information about
the files to be cloned. This DB has an index number
as key pointing to pickled information about each file.
This pickled information is
[<File Info Object>, <Host ID>, <Mount Point>]
(string)
tmpFilePat: File pattern to be used for generating temporary
files (string).
targetDiskId: ID of disk to where the files cloned should be
written (string).
Returns: Void.
"""
T = TRACE()
# Make a dictionary with the available amount of space per disk.
logger.debug("Generating dictionary with available space per disk")
availDiskSpaceDic = {}
mtRootDir = srvObj.getCfg().getRootDirectory()
if (targetDiskId):
tmpDiskInfoObjList = [ngamsDiskInfo.ngamsDiskInfo().\
read(srvObj.getDb(), targetDiskId)]
else:
tmpDiskInfoObjList = ngamsDiskUtils.\
getDiskInfoForMountedDisks(srvObj.getDb(),
srvObj.getHostId(),
mtRootDir)
for diskInfoObj in tmpDiskInfoObjList:
mtPt = diskInfoObj.getMountPoint()
diskId = diskInfoObj.getDiskId()
availDiskSpaceDic[diskId] = getDiskSpaceAvail(mtPt, format="B")
# Now simulate the execution of the clone job by going sequentially
# through the files selected for cloning and subtract the required
# disk space from the available amount for each disk.
#
# When a disk reaches the threshold for considering the disk as
# completed, it is removed from the list of available disks.
cloneListDbm = ngamsDbm.ngamsDbm(cloneListDbmName)
# This dictionary contains a list of files that have been simulated
# successfully cloned. Since they are not updated in the DB, we keep
# track of this locally.
cloneSucDbmName = tmpFilePat + "_CLONE_SUCCESS_DB"
rmFile(cloneSucDbmName + "*")
cloneSucDbm = ngamsDbm.ngamsDbm(cloneSucDbmName,
cleanUpOnDestr=1,
writePerm=1)
# This is used to keep track of the files that could not be cloned
# due to lack of space.
cloneFailDbmName = tmpFilePat + "_CLONE_FAILED_DB"
rmFile(cloneFailDbmName + "*")
cloneFailDbm = ngamsDbm.ngamsDbm(cloneFailDbmName,
cleanUpOnDestr=1,
writePerm=1)
# This is used to simulate disk completion. If a specific target
# disk is defined, all other disks should be ignored (=we set them to
# completed).
cloneSimDiskCompl = []
if (targetDiskId):
tmpDiskList = ngamsDiskUtils.\
getDiskInfoForMountedDisks(srvObj.getDb(), srvObj.getHostId(),
mtRootDir)
for idx in range(len(tmpDiskList)):
if (tmpDiskList[idx].getDiskId() != targetDiskId):
cloneSimDiskCompl.append(tmpDiskList[idx].getDiskId())
# Carry out the simulated clone process.
ngamsDiskUtils.findTargetDiskResetCache()
key = 0
while (1):
if (not cloneListDbm.hasKey(str(key))): break
fileInfo = cloneListDbm.get(str(key))
key += 1
fio = fileInfo[0]
hostName = fileInfo[1]
text = "Simulating cloning of file - File ID: %s/%d, on disk " +\
"with ID: %s on host: %s"
logger.debug(text, fio.getFileId(), fio.getFileVersion(),
fio.getDiskId(), hostName)
diskExemptList = cloneSimDiskCompl + [fio.getDiskId()]
trgDiskInfo = None
while (1):
try:
trgDiskInfo = ngamsDiskUtils.\
findTargetDisk(srvObj.getHostId(),
srvObj.getDb(), srvObj.getCfg(),
fio.getFormat(), 0,
diskExemptList, 1)
except Exception, e:
if (str(e).find("NGAMS_AL_NO_STO_SETS") != -1):
# No more candidate Target Disks for this type
# of data - this file cannot be cloned.
cloneFailDbm.addIncKey(fio)
break
# Check if a file with that ID + version is already
# stored on the selected Target Disk.
fileInDb = srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(),
fio.getFileId(),
fio.getFileVersion())
fileKey = ngamsLib.genFileKey(trgDiskInfo.getDiskId(),
fio.getFileId(),
fio.getFileVersion())
fileSimCloned = cloneSucDbm.hasKey(fileKey)
if (fileInDb or fileSimCloned):
# This file is already stored on the given disk.
# Add to the exempt list.
diskExemptList.append(trgDiskInfo.getDiskId())
else:
# OK, this disk should be OK, stop looking for a
# suitable Target Disk.
logger.debug("Found suitable disk with ID: %s/Slot ID: %s",
trgDiskInfo.getDiskId(), trgDiskInfo.getSlotId())
cloneSucDbm.add(fileKey, "")
break
# We now subtract the size of the file from the available amount of
# disk space for the selected Target Disk. When the amount of available
# space goes below the threshold defined for this NG/AMS system that
# disk is considered to be completed.
if (trgDiskInfo):
diskId = trgDiskInfo.getDiskId()
availDiskSpaceDic[diskId] -= float(fio.getFileSize())
if ((availDiskSpaceDic[diskId] / 1048576.0) < \
srvObj.getCfg().getFreeSpaceDiskChangeMb()):
cloneSimDiskCompl.append(diskId)
def handleCmd(srvObj, reqPropsObj, httpRef):
"""
Handle the CARCHIVE Command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: (fileId, filePath) tuple.
"""
T = TRACE()
# Is this NG/AMS permitted to handle Archive Requests?
logger.debug("Is this NG/AMS permitted to handle Archive Requests?")
if (not srvObj.getCfg().getAllowArchiveReq()):
errMsg = genLog("NGAMS_ER_ILL_REQ", ["Archive"])
raise Exception(errMsg)
srvObj.checkSetState("Archive Request", [NGAMS_ONLINE_STATE],
[NGAMS_IDLE_SUBSTATE, NGAMS_BUSY_SUBSTATE],
NGAMS_ONLINE_STATE,
NGAMS_BUSY_SUBSTATE,
updateDb=False)
if httpRef.command != 'POST':
raise Exception("Only POST allowed for CARCHIVE")
# Get mime-type (try to guess if not provided as an HTTP parameter).
logger.debug(
"Get mime-type (try to guess if not provided as an HTTP parameter).")
if (reqPropsObj.getMimeType() == ""):
mimeType = ngamsHighLevelLib.\
determineMimeType(srvObj.getCfg(), reqPropsObj.getFileUri())
reqPropsObj.setMimeType(mimeType)
else:
mimeType = reqPropsObj.getMimeType()
# Determine the target volume, ignoring the stream concept.
logger.debug("Determine the target volume, ignoring the stream concept.")
targDiskInfo = ngamsArchiveUtils._random_target_volume(srvObj)
if (targDiskInfo == None):
errMsg = "No disk volumes are available for ingesting any files."
raise Exception(errMsg)
reqPropsObj.setTargDiskInfo(targDiskInfo)
# Generate staging filename.
logger.debug("Generate staging filename from URI: %s",
reqPropsObj.getFileUri())
if (reqPropsObj.getFileUri().find("file_id=") >= 0):
file_id = reqPropsObj.getFileUri().split("file_id=")[1]
baseName = os.path.basename(file_id)
else:
baseName = os.path.basename(reqPropsObj.getFileUri())
stgFilename = os.path.join("/", targDiskInfo.getMountPoint(),
NGAMS_STAGING_DIR,
genUniqueId() + "___" + baseName)
logger.debug("Staging filename is: %s", stgFilename)
reqPropsObj.setStagingFilename(stgFilename)
# Retrieve file contents (from URL, archive pull, or by storing the body
# of the HTTP request, archive push).
stagingInfo = saveInStagingFile(srvObj.getCfg(), reqPropsObj, httpRef,
stgFilename, targDiskInfo)
ioTime = stagingInfo[0]
rootContainer = stagingInfo[1]
fileDataList = stagingInfo[2]
ingestRate = stagingInfo[3]
reqPropsObj.incIoTime(ioTime)
createContainers(rootContainer, None, srvObj)
parDic = {}
ngamsGenDapi.handlePars(reqPropsObj, parDic)
diskInfo = reqPropsObj.getTargDiskInfo()
# Generate file information.
logger.debug("Generate file information")
dateDir = toiso8601(fmt=FMT_DATE_ONLY)
resDapiList = []
containerSizes = {}
for item in fileDataList:
container = item[0]
filepath = item[1]
crc = item[2]
containerId = str(container.getContainerId())
basename = os.path.basename(filepath)
fileId = basename
fileVersion, relPath, relFilename,\
complFilename, fileExists =\
ngamsPlugInApi.genFileInfo(srvObj.getDb(),
srvObj.getCfg(),
reqPropsObj, diskInfo,
filepath,
fileId,
basename, [dateDir])
complFilename, relFilename = ngamsGenDapi.checkForDblExt(
complFilename, relFilename)
# Keep track of the total size of the container
uncomprSize = ngamsPlugInApi.getFileSize(filepath)
if containerId not in containerSizes:
containerSizes[containerId] = 0
containerSizes[containerId] += uncomprSize
mimeType = reqPropsObj.getMimeType()
compression = "NONE"
archFileSize = ngamsPlugInApi.getFileSize(filepath)
resDapi = ngamsPlugInApi.genDapiSuccessStat(
diskInfo.getDiskId(), relFilename, fileId, fileVersion, mimeType,
archFileSize, uncomprSize, compression, relPath,
diskInfo.getSlotId(), fileExists, complFilename)
# Move file to final destination.
logger.debug("Moving file to final destination")
ioTime = mvFile(filepath, resDapi.getCompleteFilename())
reqPropsObj.incIoTime(ioTime)
# Get crc info
checksumPlugIn = "StreamCrc32"
checksum = str(crc)
# Get source file version
# e.g.: http://ngas03.hq.eso.org:7778/RETRIEVE?file_version=1&file_id=X90/X962a4/X1
logger.debug("Get file version")
file_version = resDapi.getFileVersion()
if reqPropsObj.getFileUri().count("file_version"):
file_version = int(
(reqPropsObj.getFileUri().split("file_version=")[1]
).split("&")[0])
# Check/generate remaining file info + update in DB.
logger.debug("Creating db entry")
creDate = getFileCreationTime(resDapi.getCompleteFilename())
fileInfo = ngamsFileInfo.ngamsFileInfo().\
setDiskId(resDapi.getDiskId()).\
setFilename(resDapi.getRelFilename()).\
setFileId(resDapi.getFileId()).\
setFileVersion(file_version).\
setFormat(resDapi.getFormat()).\
setFileSize(resDapi.getFileSize()).\
setUncompressedFileSize(resDapi.getUncomprSize()).\
setCompression(resDapi.getCompression()).\
setIngestionDate(time.time()).\
setChecksum(checksum).setChecksumPlugIn(checksumPlugIn).\
setFileStatus(NGAMS_FILE_STATUS_OK).\
setCreationDate(creDate).\
setIoTime(reqPropsObj.getIoTime())
fileInfo.write(srvObj.getHostId(), srvObj.getDb())
# Add the file to the container
srvObj.getDb().addFileToContainer(containerId, resDapi.getFileId(),
True)
# Update the container sizes
for contSizeInfo in containerSizes.items():
srvObj.getDb().setContainerSize(contSizeInfo[0], contSizeInfo[1])
# Inform the caching service about the new file.
logger.debug("Inform the caching service about the new file.")
if (srvObj.getCachingActive()):
diskId = resDapi.getDiskId()
fileId = resDapi.getFileId()
fileVersion = file_version
filename = resDapi.getRelFilename()
ngamsCacheControlThread.addEntryNewFilesDbm(
srvObj, diskId, fileId, fileVersion, filename)
# Update disk info in NGAS Disks.
logger.debug("Update disk info in NGAS Disks.")
srvObj.getDb().updateDiskInfo(resDapi.getFileSize(),
resDapi.getDiskId())
resDapiList.append(resDapi)
# Check if the disk is completed.
# We use an approximate extimate for the remaning disk space to avoid
# to read the DB.
logger.debug("Check available space in disk")
availSpace = getDiskSpaceAvail(targDiskInfo.getMountPoint(), smart=False)
if (availSpace < srvObj.getCfg().getFreeSpaceDiskChangeMb()):
targDiskInfo.setCompleted(1).setCompletionDate(time.time())
targDiskInfo.write(srvObj.getDb())
# Request after-math ...
srvObj.setSubState(NGAMS_IDLE_SUBSTATE)
msg = "Successfully handled Archive Pull Request for data file " +\
"with URI: " + reqPropsObj.getSafeFileUri()
logger.info(msg)
httpRef.send_ingest_status(msg, targDiskInfo)
for resDapi in resDapiList:
# Trigger Subscription Thread. This is a special version for MWA, in which we simply swapped MIRRARCHIVE and QARCHIVE
# [email protected]
logger.debug("triggering SubscriptionThread for file %s",
resDapi.getFileId())
srvObj.addSubscriptionInfo(
[(resDapi.getFileId(), resDapi.getFileVersion())], [])
srvObj.triggerSubscriptionThread()
def _remDisk(srvObj, reqPropsObj, diskId, execute, tmpFilePat):
"""
See documentation for the ngamsRemDiskCmd.remDisk() function.
"""
# Get disk info -- if any available.
sqlDiskInfo = srvObj.getDb().getDiskInfoFromDiskId(diskId)
diskInfo = ngamsDiskInfo.ngamsDiskInfo()
if (sqlDiskInfo != []):
if (diskInfo.unpackSqlResult(sqlDiskInfo).getHostId() !=
srvObj.getHostId()):
sqlDiskInfo = None
# Check that the disk is mounted in this unit (no proxy for the REMDISK
# Command ...).
if (diskInfo.getHostId() != srvObj.getHostId()):
errMsg = "Disk referred to by Disk ID: %s seems not to be mounted " +\
"in this unit: %s -- rejecting REMDISK Command"
errMsg = errMsg % (diskId, srvObj.getHostId())
raise Exception(errMsg)
# Check that execution of the request can be granted.
filesMisCopyDbmName, filesNotRegDbmName, fileListDbmName =\
ngamsRemUtils.checkFileCopiesAndReg(srvObj, 3,
tmpFilePat, None,
diskId)
status = ngamsRemUtils._remStatErrReport(srvObj, reqPropsObj, tmpFilePat,
filesMisCopyDbmName,
filesNotRegDbmName,
fileListDbmName, diskId)
misCopiesDbm = ngamsDbm.ngamsDbm(filesMisCopyDbmName)
if (misCopiesDbm.getCount() == 0):
logger.info("Disk with ID: %s approved for deletion", diskId)
else:
logger.info("Disk with ID: %s rejected for deletion", diskId)
if (status): return status
# Check if there is enough space on the disk to store the file info
# cache files.
kbPerFile = 0.256
tmpDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm=0)
noOfFiles = tmpDbm.getCount()
del tmpDbm
kbsAvail = getDiskSpaceAvail(diskInfo.getMountPoint(), format="KB")
kbsReq = (kbPerFile * noOfFiles)
msg = "Space required for REMDISK Command: %.1f KB, " +\
"space available: %.1f KB"
logger.debug(msg, kbsReq, kbsAvail)
if (kbsReq > kbsAvail):
errMsg = "Not enough space on disk to carry out REMDISK Command. " +\
"Host: %s, Disk ID: %s, Mount Point: %s. " +\
"Required disk space: %.3f MB"
errMsg = errMsg % (getHostName(), diskInfo.getDiskId(),
diskInfo.getMountPoint(), (kbsReq / 1024.))
raise Exception(errMsg)
#########################################################################
# Execute the deletion if execute = 1 and a matching disk was found.
# Otherwise just return the ngamsStatus object as confirmation.
#########################################################################
if (execute and (sqlDiskInfo != [])):
# We delete first the info in the DB in connection with the disk and
# afterwards the contents on the disk. This is done like this, since
# it is considered worse having entries for files in the DB which are
# not available on the disk since NGAS uses the info in the DB to check
# for the number of available copies.
logger.info("Removing DB record for disk with ID: %s", diskId)
try:
tmpDir = os.path.dirname(tmpFilePat)
srvObj.getDb().deleteDiskInfo(diskId, 1)
except Exception as e:
errMsg = genLog("NGAMS_ER_DEL_DISK_DB", [diskId, str(e)])
raise Exception(errMsg)
logger.info("Deleting contents on disk with ID: %s", diskId)
try:
rmFile(os.path.normpath(diskInfo.getMountPoint() + "/*"))
except Exception as e:
errMsg = genLog("NGAMS_ER_DEL_DISK", [diskId, str(e)])
raise Exception(errMsg)
try:
# Remember to remove entry for disk from the Physical Disk Dic.
del srvObj.getDiskDic()[diskInfo.getSlotId()]
except:
pass
infoMsg = genLog("NGAMS_INFO_DEL_DISK", [diskId])
logger.info(infoMsg)
# Add entry in the NGAS Disks History Table.
ngasDiskInfo = ngamsDiskUtils.prepNgasDiskInfoFile(
srvObj.getHostId(), diskInfo, 1, 1)
srvObj.getDb().addDiskHistEntry(srvObj.getHostId(), diskId,
"Disk Removed", NGAMS_XML_MT,
ngasDiskInfo)
elif (sqlDiskInfo != []):
infoMsg = genLog("NGAMS_INFO_DEL_DISK_SEL", [diskId])
else:
infoMsg = genLog("NGAMS_WA_DEL_DISK2", [srvObj.getHostId(), diskId])
# Generate status.
status = srvObj.genStatus(NGAMS_SUCCESS, infoMsg)
if (sqlDiskInfo): status.addDiskStatus(diskInfo)
#########################################################################
return status
def handleCmd(srvObj, reqPropsObj, httpRef):
"""
Handle the Quick Archive (QARCHIVE) Command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: (fileId, filePath) tuple.
"""
T = TRACE()
# Check if the URI is correctly set.
logger.debug("Check if the URI is correctly set.")
logger.debug("ReqPropsObj status: %s", reqPropsObj.getObjStatus())
parsDic = reqPropsObj.getHttpParsDic()
if (not parsDic.has_key('fileUri') or parsDic['fileUri'] == ""):
errMsg = genLog("NGAMS_ER_MISSING_URI")
raise Exception(errMsg)
else:
reqPropsObj.setFileUri(parsDic['fileUri'])
fileUri = reqPropsObj.getFileUri()
# Is this NG/AMS permitted to handle Archive Requests?
logger.debug("Is this NG/AMS permitted to handle Archive Requests?")
if (not srvObj.getCfg().getAllowArchiveReq()):
errMsg = genLog("NGAMS_ER_ILL_REQ", ["Archive"])
raise Exception, errMsg
srvObj.checkSetState("Archive Request", [NGAMS_ONLINE_STATE],
[NGAMS_IDLE_SUBSTATE, NGAMS_BUSY_SUBSTATE],
NGAMS_ONLINE_STATE,
NGAMS_BUSY_SUBSTATE,
updateDb=False)
# Get mime-type (try to guess if not provided as an HTTP parameter).
logger.debug(
"Get mime-type (try to guess if not provided as an HTTP parameter).")
if (not parsDic.has_key('mimeType') or parsDic['mimeType'] == ""):
mimeType = ""
reqPropsObj.setMimeType("")
else:
reqPropsObj.setMimeType(parsDic['mimeType'])
mimeType = reqPropsObj.getMimeType()
if (reqPropsObj.getMimeType() == ""):
mimeType = ngamsHighLevelLib.\
determineMimeType(srvObj.getCfg(), reqPropsObj.getFileUri())
reqPropsObj.setMimeType(mimeType)
else:
mimeType = reqPropsObj.getMimeType()
ioTime = 0
reqPropsObj.incIoTime(ioTime)
(resDapi, targDiskInfo, iorate) = archiveFromFile(srvObj, fileUri, 0,
mimeType, reqPropsObj)
if (resDapi == NGAMS_FAILURE):
errMsg = targDiskInfo
httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=500)
return
# Get crc info
# info(3, "Get checksum info")
# crc = None
# TODO: Investigate how CRC calculation could be performed best.
# checksumPlugIn = "ngamsGenCrc32"
# checksum = str(crc)
# info(3, "Invoked Checksum Plug-In: " + checksumPlugIn +\
# " to handle file: " + resDapi.getCompleteFilename() +\
# ". Result: " + checksum)
# Inform the caching service about the new file.
logger.debug("Inform the caching service about the new file.")
if (srvObj.getCachingActive()):
diskId = resDapi.getDiskId()
fileId = resDapi.getFileId()
fileVersion = 1
filename = resDapi.getRelFilename()
ngamsCacheControlThread.addEntryNewFilesDbm(srvObj, diskId, fileId,
fileVersion, filename)
# Update disk info in NGAS Disks.
logger.debug("Update disk info in NGAS Disks.")
srvObj.getDb().updateDiskInfo(resDapi.getFileSize(), resDapi.getDiskId())
# Check if the disk is completed.
# We use an approximate estimate for the remaning disk space to avoid
# to read the DB.
logger.debug("Check available space in disk")
availSpace = getDiskSpaceAvail(targDiskInfo.getMountPoint(), smart=False)
if (availSpace < srvObj.getCfg().getFreeSpaceDiskChangeMb()):
targDiskInfo.setCompleted(1).setCompletionDate(time.time())
targDiskInfo.write(srvObj.getDb())
# Request after-math ...
srvObj.setSubState(NGAMS_IDLE_SUBSTATE)
msg = "Successfully handled Archive Pull Request for data file " +\
"with URI: " + reqPropsObj.getSafeFileUri()
logger.info(msg)
httpRef.send_ingest_status(msg, targDiskInfo)
# Trigger Subscription Thread. This is a special version for MWA, in which we simply swapped MIRRARCHIVE and QARCHIVE
# [email protected]
logger.debug("triggering SubscriptionThread for file %s",
resDapi.getFileId())
srvObj.addSubscriptionInfo(
[(resDapi.getFileId(), resDapi.getFileVersion())], [])
srvObj.triggerSubscriptionThread()
return (resDapi.getFileId(),
'%s/%s' % (targDiskInfo.getMountPoint(), resDapi.getRelFilename()),
iorate)