def _handleFileList(srvObj, reqPropsObj, httpRef): """ Handle STATUS?file_list... Command. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: The File List ID allocated for this request (string). """ T = TRACE() # Should a lower limit for the ingestion date be taken into account. fromIngDate = None if (reqPropsObj.hasHttpPar("from_ingestion_date")): tmpTromIngDate = reqPropsObj.getHttpPar("from_ingestion_date") fromIngDate = fromiso8601(tmpTromIngDate) # Handle the unique flag. If this is specified, only information for unique # File ID/Version pairs are returned. unique = False if (reqPropsObj.hasHttpPar("unique")): unique = int(reqPropsObj.getHttpPar("unique")) # Dump the file information needed. fileListId = genUniqueId() dbmBaseName = STATUS_FILE_LIST_DBM_TAG % fileListId fileInfoDbmName = ngamsHighLevelLib.genTmpFilename(srvObj.getCfg(), dbmBaseName) # Dump the file info from the DB. Deal with uniqueness quickly # (instead of using a new file like before) try: fileInfoDbm = ngamsDbm.ngamsDbm(fileInfoDbmName, 0, 1) fileCount = 1 unique_files = set() for f in srvObj.db.files_in_host(srvObj.getHostId(), from_date=fromIngDate): if unique: key = str('%s_%d' % (f[2], f[3])) if key in unique_files: continue else: key = str(fileCount) fileInfoDbm.add(key, f) fileCount += 1 except Exception as e: rmFile(fileInfoDbmName) msg = "Problem generating file list for STATUS Command. " +\ "Parameters: from_ingestion_date=%s. Error: %s" %\ (str(fromIngDate), str(e)) raise Exception(msg) return fileListId
def calculate_staging_name(ngams_server, file_id, existing_staging_file): """ Generate staging filename. As of v2016.06 we either have a staging filename or just the disk volume to which the file should be mirrored How can we tell which is which? staging filenames all include the character string "___" """ staging_filename = None target_disk_info = None logger.debug('Existing staging file: |%s|', existing_staging_file) if existing_staging_file == "None": logger.error( 'Trying to mirror a file where the staging file has not been set. This should no longer happen' ) elif "___" in existing_staging_file: logger.info("An existing staging file was specified: %s", existing_staging_file) if os.path.exists(existing_staging_file): logger.info("The file still exists on disk") staging_filename = existing_staging_file mount_point = staging_filename.split(NGAMS_STAGING_DIR)[0][:-1] logger.info('Staging file mount point: %s', mount_point) target_disk_info = get_mounted_disk_info(ngams_server, mount_point) logger.info('Staging file disk info: %s', str(target_disk_info)) else: logger.info("The staging file no longer exists on disk") else: logger.info( 'Staging file does not exist, only the mount directory: %s', existing_staging_file) base_name = os.path.basename(file_id) staging_filename = os.path.join("/", existing_staging_file, NGAMS_STAGING_DIR, genUniqueId() + "___" + base_name) target_disk_info = get_mounted_disk_info(ngams_server, existing_staging_file) # I do not expect this case to be used any more from v2016.06 onwards leave it here for the moment, but delete ASAP if staging_filename is None: logger.error('Reached some old code which we should not have reached') base_name = os.path.basename(file_id) target_disk_info = get_target_volume(ngams_server) staging_filename = os.path.join("/", target_disk_info.getMountPoint(), NGAMS_STAGING_DIR, genUniqueId() + "___" + base_name) return staging_filename, target_disk_info
def calculateStagingName(srvObj, fileId, existingStagingFile): # Generate staging filename. # as of v2016.06 we either have a staging filename or just the disk volume to which the file should be mirrorired. # How can we tell which is which? staging filenames all include the character string "___" stgFilename = None logger.debug('existing staging file: |%s|', existingStagingFile) if existingStagingFile == "None": logger.error( 'trying to mirror a file where the staging file has not been set. This should no longer happen' ) elif "___" in existingStagingFile: logger.info("An existing staging file was specified: %s", existingStagingFile) if os.path.exists(existingStagingFile): logger.info("the file still exists on disk") stgFilename = existingStagingFile mountPoint = stgFilename.split(NGAMS_STAGING_DIR)[0][:-1] logger.info('mount point: %s', mountPoint) targDiskInfo = getMountedDiskInfo(srvObj, mountPoint) logger.info('disk info: %s', str(targDiskInfo)) else: logger.info("the file no longer exists on disk") else: logger.info('staging file does not exist, only the mount dir: %s', existingStagingFile) baseName = os.path.basename(fileId) stgFilename = os.path.join("/", existingStagingFile, NGAMS_STAGING_DIR, genUniqueId() + "___" + baseName) targDiskInfo = getMountedDiskInfo(srvObj, existingStagingFile) # I do not expect this case to be used any more from v2016.06 onwards # leave it here for the moment, but delete ASAP if stgFilename == None: logger.error('reached some old code which we shouldnt have reached') baseName = os.path.basename(fileId) targDiskInfo = getTargetVolume(srvObj) stgFilename = os.path.join("/", targDiskInfo.getMountPoint(), NGAMS_STAGING_DIR, genUniqueId() + "___" + baseName) return (stgFilename, targDiskInfo)
def handleCmd(srvObj, reqPropsObj, httpRef): """ Handle the CARCHIVE Command. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: (fileId, filePath) tuple. """ T = TRACE() # Is this NG/AMS permitted to handle Archive Requests? logger.debug("Is this NG/AMS permitted to handle Archive Requests?") if (not srvObj.getCfg().getAllowArchiveReq()): errMsg = genLog("NGAMS_ER_ILL_REQ", ["Archive"]) raise Exception(errMsg) srvObj.checkSetState("Archive Request", [NGAMS_ONLINE_STATE], [NGAMS_IDLE_SUBSTATE, NGAMS_BUSY_SUBSTATE], NGAMS_ONLINE_STATE, NGAMS_BUSY_SUBSTATE, updateDb=False) if httpRef.command != 'POST': raise Exception("Only POST allowed for CARCHIVE") # Get mime-type (try to guess if not provided as an HTTP parameter). logger.debug( "Get mime-type (try to guess if not provided as an HTTP parameter).") if (reqPropsObj.getMimeType() == ""): mimeType = ngamsHighLevelLib.\ determineMimeType(srvObj.getCfg(), reqPropsObj.getFileUri()) reqPropsObj.setMimeType(mimeType) else: mimeType = reqPropsObj.getMimeType() # Determine the target volume, ignoring the stream concept. logger.debug("Determine the target volume, ignoring the stream concept.") targDiskInfo = ngamsArchiveUtils._random_target_volume(srvObj) if (targDiskInfo == None): errMsg = "No disk volumes are available for ingesting any files." raise Exception(errMsg) reqPropsObj.setTargDiskInfo(targDiskInfo) # Generate staging filename. logger.debug("Generate staging filename from URI: %s", reqPropsObj.getFileUri()) if (reqPropsObj.getFileUri().find("file_id=") >= 0): file_id = reqPropsObj.getFileUri().split("file_id=")[1] baseName = os.path.basename(file_id) else: baseName = os.path.basename(reqPropsObj.getFileUri()) stgFilename = os.path.join("/", targDiskInfo.getMountPoint(), NGAMS_STAGING_DIR, genUniqueId() + "___" + baseName) logger.debug("Staging filename is: %s", stgFilename) reqPropsObj.setStagingFilename(stgFilename) # Retrieve file contents (from URL, archive pull, or by storing the body # of the HTTP request, archive push). stagingInfo = saveInStagingFile(srvObj.getCfg(), reqPropsObj, httpRef, stgFilename, targDiskInfo) ioTime = stagingInfo[0] rootContainer = stagingInfo[1] fileDataList = stagingInfo[2] ingestRate = stagingInfo[3] reqPropsObj.incIoTime(ioTime) createContainers(rootContainer, None, srvObj) parDic = {} ngamsGenDapi.handlePars(reqPropsObj, parDic) diskInfo = reqPropsObj.getTargDiskInfo() # Generate file information. logger.debug("Generate file information") dateDir = toiso8601(fmt=FMT_DATE_ONLY) resDapiList = [] containerSizes = {} for item in fileDataList: container = item[0] filepath = item[1] crc = item[2] containerId = str(container.getContainerId()) basename = os.path.basename(filepath) fileId = basename fileVersion, relPath, relFilename,\ complFilename, fileExists =\ ngamsPlugInApi.genFileInfo(srvObj.getDb(), srvObj.getCfg(), reqPropsObj, diskInfo, filepath, fileId, basename, [dateDir]) complFilename, relFilename = ngamsGenDapi.checkForDblExt( complFilename, relFilename) # Keep track of the total size of the container uncomprSize = ngamsPlugInApi.getFileSize(filepath) if containerId not in containerSizes: containerSizes[containerId] = 0 containerSizes[containerId] += uncomprSize mimeType = reqPropsObj.getMimeType() compression = "NONE" archFileSize = ngamsPlugInApi.getFileSize(filepath) resDapi = ngamsPlugInApi.genDapiSuccessStat( diskInfo.getDiskId(), relFilename, fileId, fileVersion, mimeType, archFileSize, uncomprSize, compression, relPath, diskInfo.getSlotId(), fileExists, complFilename) # Move file to final destination. logger.debug("Moving file to final destination") ioTime = mvFile(filepath, resDapi.getCompleteFilename()) reqPropsObj.incIoTime(ioTime) # Get crc info checksumPlugIn = "StreamCrc32" checksum = str(crc) # Get source file version # e.g.: http://ngas03.hq.eso.org:7778/RETRIEVE?file_version=1&file_id=X90/X962a4/X1 logger.debug("Get file version") file_version = resDapi.getFileVersion() if reqPropsObj.getFileUri().count("file_version"): file_version = int( (reqPropsObj.getFileUri().split("file_version=")[1] ).split("&")[0]) # Check/generate remaining file info + update in DB. logger.debug("Creating db entry") creDate = getFileCreationTime(resDapi.getCompleteFilename()) fileInfo = ngamsFileInfo.ngamsFileInfo().\ setDiskId(resDapi.getDiskId()).\ setFilename(resDapi.getRelFilename()).\ setFileId(resDapi.getFileId()).\ setFileVersion(file_version).\ setFormat(resDapi.getFormat()).\ setFileSize(resDapi.getFileSize()).\ setUncompressedFileSize(resDapi.getUncomprSize()).\ setCompression(resDapi.getCompression()).\ setIngestionDate(time.time()).\ setChecksum(checksum).setChecksumPlugIn(checksumPlugIn).\ setFileStatus(NGAMS_FILE_STATUS_OK).\ setCreationDate(creDate).\ setIoTime(reqPropsObj.getIoTime()) fileInfo.write(srvObj.getHostId(), srvObj.getDb()) # Add the file to the container srvObj.getDb().addFileToContainer(containerId, resDapi.getFileId(), True) # Update the container sizes for contSizeInfo in containerSizes.items(): srvObj.getDb().setContainerSize(contSizeInfo[0], contSizeInfo[1]) # Inform the caching service about the new file. logger.debug("Inform the caching service about the new file.") if (srvObj.getCachingActive()): diskId = resDapi.getDiskId() fileId = resDapi.getFileId() fileVersion = file_version filename = resDapi.getRelFilename() ngamsCacheControlThread.addEntryNewFilesDbm( srvObj, diskId, fileId, fileVersion, filename) # Update disk info in NGAS Disks. logger.debug("Update disk info in NGAS Disks.") srvObj.getDb().updateDiskInfo(resDapi.getFileSize(), resDapi.getDiskId()) resDapiList.append(resDapi) # Check if the disk is completed. # We use an approximate extimate for the remaning disk space to avoid # to read the DB. logger.debug("Check available space in disk") availSpace = getDiskSpaceAvail(targDiskInfo.getMountPoint(), smart=False) if (availSpace < srvObj.getCfg().getFreeSpaceDiskChangeMb()): targDiskInfo.setCompleted(1).setCompletionDate(time.time()) targDiskInfo.write(srvObj.getDb()) # Request after-math ... srvObj.setSubState(NGAMS_IDLE_SUBSTATE) msg = "Successfully handled Archive Pull Request for data file " +\ "with URI: " + reqPropsObj.getSafeFileUri() logger.info(msg) httpRef.send_ingest_status(msg, targDiskInfo) for resDapi in resDapiList: # Trigger Subscription Thread. This is a special version for MWA, in which we simply swapped MIRRARCHIVE and QARCHIVE # [email protected] logger.debug("triggering SubscriptionThread for file %s", resDapi.getFileId()) srvObj.addSubscriptionInfo( [(resDapi.getFileId(), resDapi.getFileVersion())], []) srvObj.triggerSubscriptionThread()