def _handleFileList(srvObj, reqPropsObj, httpRef):
"""
Handle STATUS?file_list... Command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: The File List ID allocated for this request (string).
"""
T = TRACE()
# Should a lower limit for the ingestion date be taken into account.
fromIngDate = None
if (reqPropsObj.hasHttpPar("from_ingestion_date")):
tmpTromIngDate = reqPropsObj.getHttpPar("from_ingestion_date")
fromIngDate = fromiso8601(tmpTromIngDate)
# Handle the unique flag. If this is specified, only information for unique
# File ID/Version pairs are returned.
unique = False
if (reqPropsObj.hasHttpPar("unique")):
unique = int(reqPropsObj.getHttpPar("unique"))
# Dump the file information needed.
fileListId = genUniqueId()
dbmBaseName = STATUS_FILE_LIST_DBM_TAG % fileListId
fileInfoDbmName = ngamsHighLevelLib.genTmpFilename(srvObj.getCfg(),
dbmBaseName)
# Dump the file info from the DB. Deal with uniqueness quickly
# (instead of using a new file like before)
try:
fileInfoDbm = ngamsDbm.ngamsDbm(fileInfoDbmName, 0, 1)
fileCount = 1
unique_files = set()
for f in srvObj.db.files_in_host(srvObj.getHostId(),
from_date=fromIngDate):
if unique:
key = str('%s_%d' % (f[2], f[3]))
if key in unique_files:
continue
else:
key = str(fileCount)
fileInfoDbm.add(key, f)
fileCount += 1
except Exception as e:
rmFile(fileInfoDbmName)
msg = "Problem generating file list for STATUS Command. " +\
"Parameters: from_ingestion_date=%s. Error: %s" %\
(str(fromIngDate), str(e))
raise Exception(msg)
return fileListId
def calculate_staging_name(ngams_server, file_id, existing_staging_file):
"""
Generate staging filename.
As of v2016.06 we either have a staging filename or just the disk volume to which the file should be mirrored
How can we tell which is which? staging filenames all include the character string "___"
"""
staging_filename = None
target_disk_info = None
logger.debug('Existing staging file: |%s|', existing_staging_file)
if existing_staging_file == "None":
logger.error(
'Trying to mirror a file where the staging file has not been set. This should no longer happen'
)
elif "___" in existing_staging_file:
logger.info("An existing staging file was specified: %s",
existing_staging_file)
if os.path.exists(existing_staging_file):
logger.info("The file still exists on disk")
staging_filename = existing_staging_file
mount_point = staging_filename.split(NGAMS_STAGING_DIR)[0][:-1]
logger.info('Staging file mount point: %s', mount_point)
target_disk_info = get_mounted_disk_info(ngams_server, mount_point)
logger.info('Staging file disk info: %s', str(target_disk_info))
else:
logger.info("The staging file no longer exists on disk")
else:
logger.info(
'Staging file does not exist, only the mount directory: %s',
existing_staging_file)
base_name = os.path.basename(file_id)
staging_filename = os.path.join("/", existing_staging_file,
NGAMS_STAGING_DIR,
genUniqueId() + "___" + base_name)
target_disk_info = get_mounted_disk_info(ngams_server,
existing_staging_file)
# I do not expect this case to be used any more from v2016.06 onwards leave it here for the moment, but delete ASAP
if staging_filename is None:
logger.error('Reached some old code which we should not have reached')
base_name = os.path.basename(file_id)
target_disk_info = get_target_volume(ngams_server)
staging_filename = os.path.join("/", target_disk_info.getMountPoint(),
NGAMS_STAGING_DIR,
genUniqueId() + "___" + base_name)
return staging_filename, target_disk_info
def calculateStagingName(srvObj, fileId, existingStagingFile):
# Generate staging filename.
# as of v2016.06 we either have a staging filename or just the disk volume to which the file should be mirrorired.
# How can we tell which is which? staging filenames all include the character string "___"
stgFilename = None
logger.debug('existing staging file: |%s|', existingStagingFile)
if existingStagingFile == "None":
logger.error(
'trying to mirror a file where the staging file has not been set. This should no longer happen'
)
elif "___" in existingStagingFile:
logger.info("An existing staging file was specified: %s",
existingStagingFile)
if os.path.exists(existingStagingFile):
logger.info("the file still exists on disk")
stgFilename = existingStagingFile
mountPoint = stgFilename.split(NGAMS_STAGING_DIR)[0][:-1]
logger.info('mount point: %s', mountPoint)
targDiskInfo = getMountedDiskInfo(srvObj, mountPoint)
logger.info('disk info: %s', str(targDiskInfo))
else:
logger.info("the file no longer exists on disk")
else:
logger.info('staging file does not exist, only the mount dir: %s',
existingStagingFile)
baseName = os.path.basename(fileId)
stgFilename = os.path.join("/", existingStagingFile, NGAMS_STAGING_DIR,
genUniqueId() + "___" + baseName)
targDiskInfo = getMountedDiskInfo(srvObj, existingStagingFile)
# I do not expect this case to be used any more from v2016.06 onwards
# leave it here for the moment, but delete ASAP
if stgFilename == None:
logger.error('reached some old code which we shouldnt have reached')
baseName = os.path.basename(fileId)
targDiskInfo = getTargetVolume(srvObj)
stgFilename = os.path.join("/", targDiskInfo.getMountPoint(),
NGAMS_STAGING_DIR,
genUniqueId() + "___" + baseName)
return (stgFilename, targDiskInfo)
def handleCmd(srvObj, reqPropsObj, httpRef):
"""
Handle the CARCHIVE Command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: (fileId, filePath) tuple.
"""
T = TRACE()
# Is this NG/AMS permitted to handle Archive Requests?
logger.debug("Is this NG/AMS permitted to handle Archive Requests?")
if (not srvObj.getCfg().getAllowArchiveReq()):
errMsg = genLog("NGAMS_ER_ILL_REQ", ["Archive"])
raise Exception(errMsg)
srvObj.checkSetState("Archive Request", [NGAMS_ONLINE_STATE],
[NGAMS_IDLE_SUBSTATE, NGAMS_BUSY_SUBSTATE],
NGAMS_ONLINE_STATE,
NGAMS_BUSY_SUBSTATE,
updateDb=False)
if httpRef.command != 'POST':
raise Exception("Only POST allowed for CARCHIVE")
# Get mime-type (try to guess if not provided as an HTTP parameter).
logger.debug(
"Get mime-type (try to guess if not provided as an HTTP parameter).")
if (reqPropsObj.getMimeType() == ""):
mimeType = ngamsHighLevelLib.\
determineMimeType(srvObj.getCfg(), reqPropsObj.getFileUri())
reqPropsObj.setMimeType(mimeType)
else:
mimeType = reqPropsObj.getMimeType()
# Determine the target volume, ignoring the stream concept.
logger.debug("Determine the target volume, ignoring the stream concept.")
targDiskInfo = ngamsArchiveUtils._random_target_volume(srvObj)
if (targDiskInfo == None):
errMsg = "No disk volumes are available for ingesting any files."
raise Exception(errMsg)
reqPropsObj.setTargDiskInfo(targDiskInfo)
# Generate staging filename.
logger.debug("Generate staging filename from URI: %s",
reqPropsObj.getFileUri())
if (reqPropsObj.getFileUri().find("file_id=") >= 0):
file_id = reqPropsObj.getFileUri().split("file_id=")[1]
baseName = os.path.basename(file_id)
else:
baseName = os.path.basename(reqPropsObj.getFileUri())
stgFilename = os.path.join("/", targDiskInfo.getMountPoint(),
NGAMS_STAGING_DIR,
genUniqueId() + "___" + baseName)
logger.debug("Staging filename is: %s", stgFilename)
reqPropsObj.setStagingFilename(stgFilename)
# Retrieve file contents (from URL, archive pull, or by storing the body
# of the HTTP request, archive push).
stagingInfo = saveInStagingFile(srvObj.getCfg(), reqPropsObj, httpRef,
stgFilename, targDiskInfo)
ioTime = stagingInfo[0]
rootContainer = stagingInfo[1]
fileDataList = stagingInfo[2]
ingestRate = stagingInfo[3]
reqPropsObj.incIoTime(ioTime)
createContainers(rootContainer, None, srvObj)
parDic = {}
ngamsGenDapi.handlePars(reqPropsObj, parDic)
diskInfo = reqPropsObj.getTargDiskInfo()
# Generate file information.
logger.debug("Generate file information")
dateDir = toiso8601(fmt=FMT_DATE_ONLY)
resDapiList = []
containerSizes = {}
for item in fileDataList:
container = item[0]
filepath = item[1]
crc = item[2]
containerId = str(container.getContainerId())
basename = os.path.basename(filepath)
fileId = basename
fileVersion, relPath, relFilename,\
complFilename, fileExists =\
ngamsPlugInApi.genFileInfo(srvObj.getDb(),
srvObj.getCfg(),
reqPropsObj, diskInfo,
filepath,
fileId,
basename, [dateDir])
complFilename, relFilename = ngamsGenDapi.checkForDblExt(
complFilename, relFilename)
# Keep track of the total size of the container
uncomprSize = ngamsPlugInApi.getFileSize(filepath)
if containerId not in containerSizes:
containerSizes[containerId] = 0
containerSizes[containerId] += uncomprSize
mimeType = reqPropsObj.getMimeType()
compression = "NONE"
archFileSize = ngamsPlugInApi.getFileSize(filepath)
resDapi = ngamsPlugInApi.genDapiSuccessStat(
diskInfo.getDiskId(), relFilename, fileId, fileVersion, mimeType,
archFileSize, uncomprSize, compression, relPath,
diskInfo.getSlotId(), fileExists, complFilename)
# Move file to final destination.
logger.debug("Moving file to final destination")
ioTime = mvFile(filepath, resDapi.getCompleteFilename())
reqPropsObj.incIoTime(ioTime)
# Get crc info
checksumPlugIn = "StreamCrc32"
checksum = str(crc)
# Get source file version
# e.g.: http://ngas03.hq.eso.org:7778/RETRIEVE?file_version=1&file_id=X90/X962a4/X1
logger.debug("Get file version")
file_version = resDapi.getFileVersion()
if reqPropsObj.getFileUri().count("file_version"):
file_version = int(
(reqPropsObj.getFileUri().split("file_version=")[1]
).split("&")[0])
# Check/generate remaining file info + update in DB.
logger.debug("Creating db entry")
creDate = getFileCreationTime(resDapi.getCompleteFilename())
fileInfo = ngamsFileInfo.ngamsFileInfo().\
setDiskId(resDapi.getDiskId()).\
setFilename(resDapi.getRelFilename()).\
setFileId(resDapi.getFileId()).\
setFileVersion(file_version).\
setFormat(resDapi.getFormat()).\
setFileSize(resDapi.getFileSize()).\
setUncompressedFileSize(resDapi.getUncomprSize()).\
setCompression(resDapi.getCompression()).\
setIngestionDate(time.time()).\
setChecksum(checksum).setChecksumPlugIn(checksumPlugIn).\
setFileStatus(NGAMS_FILE_STATUS_OK).\
setCreationDate(creDate).\
setIoTime(reqPropsObj.getIoTime())
fileInfo.write(srvObj.getHostId(), srvObj.getDb())
# Add the file to the container
srvObj.getDb().addFileToContainer(containerId, resDapi.getFileId(),
True)
# Update the container sizes
for contSizeInfo in containerSizes.items():
srvObj.getDb().setContainerSize(contSizeInfo[0], contSizeInfo[1])
# Inform the caching service about the new file.
logger.debug("Inform the caching service about the new file.")
if (srvObj.getCachingActive()):
diskId = resDapi.getDiskId()
fileId = resDapi.getFileId()
fileVersion = file_version
filename = resDapi.getRelFilename()
ngamsCacheControlThread.addEntryNewFilesDbm(
srvObj, diskId, fileId, fileVersion, filename)
# Update disk info in NGAS Disks.
logger.debug("Update disk info in NGAS Disks.")
srvObj.getDb().updateDiskInfo(resDapi.getFileSize(),
resDapi.getDiskId())
resDapiList.append(resDapi)
# Check if the disk is completed.
# We use an approximate extimate for the remaning disk space to avoid
# to read the DB.
logger.debug("Check available space in disk")
availSpace = getDiskSpaceAvail(targDiskInfo.getMountPoint(), smart=False)
if (availSpace < srvObj.getCfg().getFreeSpaceDiskChangeMb()):
targDiskInfo.setCompleted(1).setCompletionDate(time.time())
targDiskInfo.write(srvObj.getDb())
# Request after-math ...
srvObj.setSubState(NGAMS_IDLE_SUBSTATE)
msg = "Successfully handled Archive Pull Request for data file " +\
"with URI: " + reqPropsObj.getSafeFileUri()
logger.info(msg)
httpRef.send_ingest_status(msg, targDiskInfo)
for resDapi in resDapiList:
# Trigger Subscription Thread. This is a special version for MWA, in which we simply swapped MIRRARCHIVE and QARCHIVE
# [email protected]
logger.debug("triggering SubscriptionThread for file %s",
resDapi.getFileId())
srvObj.addSubscriptionInfo(
[(resDapi.getFileId(), resDapi.getFileVersion())], [])
srvObj.triggerSubscriptionThread()