def receiveData(srvObj,
reqPropsObj,
httpRef):
"""
Receive the data in connection with the Rearchive Request.
For a description of the parameters: Check handleCmd().
Returns: Tuple with File Info Object for the file to be rearchived and
Disk Info Object for the selected target disk
(tuple/(fileInfo, ngamsDiskInfo)).
"""
# Note, this algorithm does not implement support for back-log buffering
# for speed optimization reasons.
# Unpack the file information contained in the HTTP header
# (NGAS-File-Info).
encFileInfo = reqPropsObj.getHttpHdr(NGAMS_HTTP_HDR_FILE_INFO)
if (not encFileInfo):
msg = "Error. Must provide NGAS File Information " +\
"(RFC 3548 encoded) in HTTP header NGAS-File-Info"
raise Exception(msg)
fileInfoXml = base64.b64decode(encFileInfo)
fileInfoObj = ngamsFileInfo.ngamsFileInfo().unpackXmlDoc(fileInfoXml)
# Find the most suitable target Disk Set.
trgDiskInfoObj = None
diskExList = []
while (True):
try:
tmpTrgDiskInfo = ngamsDiskUtils.\
findTargetDisk(srvObj.getHostId(),
srvObj.getDb(), srvObj.getCfg(),
reqPropsObj.getMimeType(),
sendNotification = 0,
diskExemptList = diskExList,
reqSpace = reqPropsObj.getSize())
if (srvObj.getDb().fileInDb(tmpTrgDiskInfo.getDiskId(),
fileInfoObj.getFileId(),
fileInfoObj.getFileVersion())):
diskExList.append(tmpTrgDiskInfo.getDiskId())
else:
trgDiskInfoObj = tmpTrgDiskInfo
break
except Exception as e:
msg = "Error locating target disk for REARCHIVE Command. Error: %s"
msg = msg % str(e)
raise Exception(msg)
# Generate Staging Filename + save file into this.
storageSetId = trgDiskInfoObj.getStorageSetId()
stagingFilename = ngamsHighLevelLib.\
genStagingFilename(srvObj.getCfg(), reqPropsObj,
srvObj.getDiskDic(), storageSetId,
reqPropsObj.getFileUri(),
genTmpFiles = 0)
reqPropsObj.setStagingFilename(stagingFilename)
# If it is an Rearchive Pull Request, open the URL.
if (reqPropsObj.getHttpMethod() == NGAMS_HTTP_GET):
# urllib.urlopen will attempt to get the content-length based on the URI
# i.e. file, ftp, http
handle = urllib.urlopen(reqPropsObj.getFileUri())
reqPropsObj.setSize(handle.info()['Content-Length'])
rfile = handle
else:
reqPropsObj.setSize(fileInfoObj.getFileSize())
rfile = httpRef.rfile
# Save the data into the Staging File.
try:
# Make mutual exclusion on disk access (if requested).
ngamsHighLevelLib.acquireDiskResource(srvObj.getCfg(), trgDiskInfoObj.getSlotId())
# If provided with a checksum, calculate the checksum on the incoming
# data stream and check that it is the same as the expected one
# (as indicated in the file info structure given by the user)
stored_checksum = fileInfoObj.getChecksum()
crc_variant = fileInfoObj.getChecksumPlugIn()
skip_crc = True
if stored_checksum and crc_variant:
reqPropsObj.addHttpPar('crc_variant', crc_variant)
reqPropsObj.__httpHdrDic[NGAMS_HTTP_HDR_CHECKSUM] = stored_checksum
skip_crc = False
ngamsArchiveUtils.archive_contents_from_request(stagingFilename, srvObj.getCfg(),
reqPropsObj, rfile, skip_crc=skip_crc)
finally:
ngamsHighLevelLib.releaseDiskResource(srvObj.getCfg(), trgDiskInfoObj.getSlotId())
# Synchronize the file caches to ensure the files have been stored
# on the disk and check that the files are accessible.
# This sync is only relevant if back-log buffering is on.
if (srvObj.getCfg().getBackLogBuffering()):
ngamsFileUtils.syncCachesCheckFiles(srvObj, [stagingFilename])
return (fileInfoObj, trgDiskInfoObj)
def saveFromHttpToFile(ngamsCfgObj,
reqPropsObj,
httpRef,
trgFilename,
blockSize,
mutexDiskAccess=1,
diskInfoObj=None):
"""
Save the data available on an HTTP channel into the given file.
ngamsCfgObj: NG/AMS Configuration object (ngamsConfig).
reqPropsObj: NG/AMS Request Properties object (ngamsReqProps).
trgFilename: Target name for file where data will be
written (string).
blockSize: Block size (bytes) to apply when reading the data
from the HTTP channel (integer).
mutexDiskAccess: Require mutual exclusion for disk access (integer).
diskInfoObj: Disk info object. Only needed if mutual exclusion
is required for disk access (ngamsDiskInfo).
Returns: Tuple. Element 0: Time in took to write
file (s) (tuple).
"""
T = TRACE()
checkCreatePath(os.path.dirname(trgFilename))
start = time.time()
try:
# Make mutual exclusion on disk access (if requested).
if (mutexDiskAccess):
ngamsHighLevelLib.acquireDiskResource(ngamsCfgObj,
diskInfoObj.getSlotId())
# Distinguish between Archive Pull and Push Request. By Archive
# Pull we may simply read the file descriptor until it returns "".
remSize = reqPropsObj.getSize()
logger.debug("Archive Push/Pull Request - Data size: %d", remSize)
handler = ngamsMIMEMultipart.FilesystemWriterHandler(
blockSize, True, trgFilename)
parser = ngamsMIMEMultipart.MIMEMultipartParser(
handler, httpRef.rfile, remSize, blockSize)
parser.parse()
deltaTime = time.time() - start
fileDataList = handler.getFileDataList()
crcTime = handler.getCrcTime()
writingTime = handler.getWritingTime()
rootContainer = handler.getRoot()
readingTime = parser.getReadingTime()
bytesRead = parser.getBytesRead()
ingestRate = (float(bytesRead) / deltaTime)
reqPropsObj.setBytesReceived(bytesRead)
logger.debug(
"Transfer time: %.3f s; CRC time: %.3f s; write time %.3f s",
readingTime, crcTime, writingTime)
return [deltaTime, rootContainer, fileDataList, ingestRate]
finally:
# Release disk resouce.
if (mutexDiskAccess):
ngamsHighLevelLib.releaseDiskResource(ngamsCfgObj,
diskInfoObj.getSlotId())
# If it is an Rearchive Pull Request, open the URL.
if (reqPropsObj.getHttpMethod() == NGAMS_HTTP_GET):
# urllib.urlopen will attempt to get the content-length based on the URI
# i.e. file, ftp, http
handle = urllib.urlopen(reqPropsObj.getFileUri())
reqPropsObj.setSize(handle.info()['Content-Length'])
rfile = handle
else:
reqPropsObj.setSize(fileInfoObj.getFileSize())
rfile = httpRef.rfile
# Save the data into the Staging File.
try:
# Make mutual exclusion on disk access (if requested).
ngamsHighLevelLib.acquireDiskResource(srvObj.getCfg(), trgDiskInfoObj.getSlotId())
# If provided with a checksum, calculate the checksum on the incoming
# data stream and check that it is the same as the expected one
# (as indicated in the file info structure given by the user)
stored_checksum = fileInfoObj.getChecksum()
crc_variant = fileInfoObj.getChecksumPlugIn()
skip_crc = True
if stored_checksum and crc_variant:
reqPropsObj.addHttpPar('crc_variant', crc_variant)
reqPropsObj.__httpHdrDic[NGAMS_HTTP_HDR_CHECKSUM] = stored_checksum
skip_crc = False
ngamsArchiveUtils.archive_contents_from_request(stagingFilename, srvObj.getCfg(),
reqPropsObj, rfile, skip_crc=skip_crc)
finally: