def receiveData(srvObj, reqPropsObj, httpRef): """ Receive the data in connection with the Rearchive Request. For a description of the parameters: Check handleCmd(). Returns: Tuple with File Info Object for the file to be rearchived and Disk Info Object for the selected target disk (tuple/(fileInfo, ngamsDiskInfo)). """ # Note, this algorithm does not implement support for back-log buffering # for speed optimization reasons. # Unpack the file information contained in the HTTP header # (NGAS-File-Info). encFileInfo = reqPropsObj.getHttpHdr(NGAMS_HTTP_HDR_FILE_INFO) if (not encFileInfo): msg = "Error. Must provide NGAS File Information " +\ "(RFC 3548 encoded) in HTTP header NGAS-File-Info" raise Exception(msg) fileInfoXml = base64.b64decode(encFileInfo) fileInfoObj = ngamsFileInfo.ngamsFileInfo().unpackXmlDoc(fileInfoXml) # Find the most suitable target Disk Set. trgDiskInfoObj = None diskExList = [] while (True): try: tmpTrgDiskInfo = ngamsDiskUtils.\ findTargetDisk(srvObj.getHostId(), srvObj.getDb(), srvObj.getCfg(), reqPropsObj.getMimeType(), sendNotification = 0, diskExemptList = diskExList, reqSpace = reqPropsObj.getSize()) if (srvObj.getDb().fileInDb(tmpTrgDiskInfo.getDiskId(), fileInfoObj.getFileId(), fileInfoObj.getFileVersion())): diskExList.append(tmpTrgDiskInfo.getDiskId()) else: trgDiskInfoObj = tmpTrgDiskInfo break except Exception as e: msg = "Error locating target disk for REARCHIVE Command. Error: %s" msg = msg % str(e) raise Exception(msg) # Generate Staging Filename + save file into this. storageSetId = trgDiskInfoObj.getStorageSetId() stagingFilename = ngamsHighLevelLib.\ genStagingFilename(srvObj.getCfg(), reqPropsObj, srvObj.getDiskDic(), storageSetId, reqPropsObj.getFileUri(), genTmpFiles = 0) reqPropsObj.setStagingFilename(stagingFilename) # If it is an Rearchive Pull Request, open the URL. if (reqPropsObj.getHttpMethod() == NGAMS_HTTP_GET): # urllib.urlopen will attempt to get the content-length based on the URI # i.e. file, ftp, http handle = urllib.urlopen(reqPropsObj.getFileUri()) reqPropsObj.setSize(handle.info()['Content-Length']) rfile = handle else: reqPropsObj.setSize(fileInfoObj.getFileSize()) rfile = httpRef.rfile # Save the data into the Staging File. try: # Make mutual exclusion on disk access (if requested). ngamsHighLevelLib.acquireDiskResource(srvObj.getCfg(), trgDiskInfoObj.getSlotId()) # If provided with a checksum, calculate the checksum on the incoming # data stream and check that it is the same as the expected one # (as indicated in the file info structure given by the user) stored_checksum = fileInfoObj.getChecksum() crc_variant = fileInfoObj.getChecksumPlugIn() skip_crc = True if stored_checksum and crc_variant: reqPropsObj.addHttpPar('crc_variant', crc_variant) reqPropsObj.__httpHdrDic[NGAMS_HTTP_HDR_CHECKSUM] = stored_checksum skip_crc = False ngamsArchiveUtils.archive_contents_from_request(stagingFilename, srvObj.getCfg(), reqPropsObj, rfile, skip_crc=skip_crc) finally: ngamsHighLevelLib.releaseDiskResource(srvObj.getCfg(), trgDiskInfoObj.getSlotId()) # Synchronize the file caches to ensure the files have been stored # on the disk and check that the files are accessible. # This sync is only relevant if back-log buffering is on. if (srvObj.getCfg().getBackLogBuffering()): ngamsFileUtils.syncCachesCheckFiles(srvObj, [stagingFilename]) return (fileInfoObj, trgDiskInfoObj)
def saveFromHttpToFile(ngamsCfgObj, reqPropsObj, httpRef, trgFilename, blockSize, mutexDiskAccess=1, diskInfoObj=None): """ Save the data available on an HTTP channel into the given file. ngamsCfgObj: NG/AMS Configuration object (ngamsConfig). reqPropsObj: NG/AMS Request Properties object (ngamsReqProps). trgFilename: Target name for file where data will be written (string). blockSize: Block size (bytes) to apply when reading the data from the HTTP channel (integer). mutexDiskAccess: Require mutual exclusion for disk access (integer). diskInfoObj: Disk info object. Only needed if mutual exclusion is required for disk access (ngamsDiskInfo). Returns: Tuple. Element 0: Time in took to write file (s) (tuple). """ T = TRACE() checkCreatePath(os.path.dirname(trgFilename)) start = time.time() try: # Make mutual exclusion on disk access (if requested). if (mutexDiskAccess): ngamsHighLevelLib.acquireDiskResource(ngamsCfgObj, diskInfoObj.getSlotId()) # Distinguish between Archive Pull and Push Request. By Archive # Pull we may simply read the file descriptor until it returns "". remSize = reqPropsObj.getSize() logger.debug("Archive Push/Pull Request - Data size: %d", remSize) handler = ngamsMIMEMultipart.FilesystemWriterHandler( blockSize, True, trgFilename) parser = ngamsMIMEMultipart.MIMEMultipartParser( handler, httpRef.rfile, remSize, blockSize) parser.parse() deltaTime = time.time() - start fileDataList = handler.getFileDataList() crcTime = handler.getCrcTime() writingTime = handler.getWritingTime() rootContainer = handler.getRoot() readingTime = parser.getReadingTime() bytesRead = parser.getBytesRead() ingestRate = (float(bytesRead) / deltaTime) reqPropsObj.setBytesReceived(bytesRead) logger.debug( "Transfer time: %.3f s; CRC time: %.3f s; write time %.3f s", readingTime, crcTime, writingTime) return [deltaTime, rootContainer, fileDataList, ingestRate] finally: # Release disk resouce. if (mutexDiskAccess): ngamsHighLevelLib.releaseDiskResource(ngamsCfgObj, diskInfoObj.getSlotId())
# If it is an Rearchive Pull Request, open the URL. if (reqPropsObj.getHttpMethod() == NGAMS_HTTP_GET): # urllib.urlopen will attempt to get the content-length based on the URI # i.e. file, ftp, http handle = urllib.urlopen(reqPropsObj.getFileUri()) reqPropsObj.setSize(handle.info()['Content-Length']) rfile = handle else: reqPropsObj.setSize(fileInfoObj.getFileSize()) rfile = httpRef.rfile # Save the data into the Staging File. try: # Make mutual exclusion on disk access (if requested). ngamsHighLevelLib.acquireDiskResource(srvObj.getCfg(), trgDiskInfoObj.getSlotId()) # If provided with a checksum, calculate the checksum on the incoming # data stream and check that it is the same as the expected one # (as indicated in the file info structure given by the user) stored_checksum = fileInfoObj.getChecksum() crc_variant = fileInfoObj.getChecksumPlugIn() skip_crc = True if stored_checksum and crc_variant: reqPropsObj.addHttpPar('crc_variant', crc_variant) reqPropsObj.__httpHdrDic[NGAMS_HTTP_HDR_CHECKSUM] = stored_checksum skip_crc = False ngamsArchiveUtils.archive_contents_from_request(stagingFilename, srvObj.getCfg(), reqPropsObj, rfile, skip_crc=skip_crc) finally: