def checkStagingAreas(srvObj):
"""
Check the Staging Areas of the disks registered, and in case files
are found on these move htese to the Back-Log Area.
srvObj: Reference to NG/AMS server class object (ngamsServer).
Returns: Void.
"""
T = TRACE()
diskList = ngamsDiskUtils.\
getDiskInfoForMountedDisks(srvObj.getDb(), srvObj.getHostId(),
srvObj.getCfg().\
getRootDirectory())
# Generate first a dictionary with all files in the staging areas.
stagingFileDic = {}
for disk in diskList:
stagingArea = disk.getStagingArea()
if (stagingArea != ""):
fileList = glob.glob(stagingArea + "/*")
fileList += glob.glob(stagingArea + "/.NGAMS*")
for filename in fileList:
stagingFileDic[filename] = 1
# Go through all files in the staging file dictionary and move them to
# the Bad Files Area.
for filename in stagingFileDic.keys():
ngamsHighLevelLib.moveFile2BadDir(srvObj.getCfg(), filename)
def checkStagingAreas(srvObj):
"""
Check the Staging Areas of the disks registered, and in case files
are found on these move htese to the Back-Log Area.
srvObj: Reference to NG/AMS server class object (ngamsServer).
Returns: Void.
"""
diskList = ngamsDiskUtils.\
getDiskInfoForMountedDisks(srvObj.getDb(), srvObj.getHostId(),
srvObj.getCfg().\
getRootDirectory())
for disk in diskList:
stagingArea = disk.getStagingArea()
if stagingArea:
files_to_move = glob.glob(stagingArea + "/*")
files_to_move += glob.glob(stagingArea + "/.NGAMS*")
for f in files_to_move:
ngamsHighLevelLib.moveFile2BadDir(disk, f)
ngamsArchiveUtils.postFileRecepHandling(srvObj, reqPropsObjLoc,
resMain, trgDiskInfo)
except Exception, e:
# If another error occurs, than one qualifying for Back-Log
# Buffering the file, we have to log an error.
if (ngamsHighLevelLib.performBackLogBuffering(srvObj.getCfg(),
reqPropsObjLoc, e)):
logger.exception(
"Tried to archive local file %s, keeping original file",
filename)
return [NGAMS_FAILURE, str(e), NGAMS_FAILURE]
else:
logger.exception("Tried to archive local file %s, " + \
"moving it to Bad Files Directory " + \
"-- cannot be handled", filename)
ngamsHighLevelLib.moveFile2BadDir(srvObj.getCfg(), filename)
# Remove pickle file if available.
pickleObjFile = filename + "." + NGAMS_PICKLE_FILE_EXT
if (os.path.exists(pickleObjFile)):
logger.debug("Removing Back-Log Buffer Pickle File: %s",
pickleObjFile)
rmFile(pickleObjFile)
return [NGAMS_FAILURE, str(e), NGAMS_FAILURE]
# If the file was handled successfully, we remove it from the
# Back-Log Buffer Directory unless the local file was a log-file
# in which case we leave the cleanup to the Janitor-Thread.
if stagingFile.find('LOG-ROTATE') > -1:
logger.debug("Successfully archived local file: %s", filename)
else:
logger.debug(
def archiveFromFile(srvObj,
filename,
noReplication=0,
mimeType=None,
reqPropsObj=None):
"""
Archive a file directly from a file as source.
srvObj: Reference to NG/AMS Server Object (ngamsServer).
filename: Name of file to archive (string).
noReplication: Flag to enable/disable replication (integer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
Returns: Execution result object of DAPI
"""
T = TRACE()
logger.debug("Archiving file: %s", filename)
logger.debug("Mimetype used is %s", mimeType)
if (reqPropsObj):
logger.debug("Request Properties Object given - using this")
reqPropsObjLoc = reqPropsObj
else:
logger.debug("No Request Properties Object given - creating one")
reqPropsObjLoc = ngamsArchiveUtils.ngamsReqProps.ngamsReqProps()
stagingFile = filename
try:
if (mimeType == None):
mimeType = ngamsHighLevelLib.determineMimeType(
srvObj.getCfg(), filename)
archive_start = time.time()
# Prepare dummy ngamsReqProps object (if an object was not given).
if (not reqPropsObj):
reqPropsObjLoc.setMimeType(mimeType)
reqPropsObjLoc.setStagingFilename(filename)
reqPropsObjLoc.setHttpMethod(NGAMS_HTTP_GET)
reqPropsObjLoc.setCmd(NGAMS_ARCHIVE_CMD)
reqPropsObjLoc.setSize(os.path.getsize(filename))
reqPropsObjLoc.setFileUri(NGAMS_HTTP_FILE_URL + filename)
reqPropsObjLoc.setNoReplication(noReplication)
# If no target disk is defined, find one suitable disk.
if (not reqPropsObjLoc.getTargDiskInfo()):
try:
trgDiskInfo = ngamsArchiveUtils.ngamsDiskUtils.\
findTargetDisk(srvObj.getHostId(),
srvObj.getDb(), srvObj.getCfg(),
mimeType, 0,
reqSpace=reqPropsObjLoc.getSize())
reqPropsObjLoc.setTargDiskInfo(trgDiskInfo)
# copy the file to the staging area of the target disk
stagingFile = trgDiskInfo.getMountPoint(
) + '/staging/' + os.path.basename(filename)
cpFile(filename, stagingFile)
reqPropsObjLoc.setStagingFilename(stagingFile)
except Exception as e:
errMsg = str(e) + ". Attempting to archive local file: " +\
filename
ngamsPlugInApi.notify(srvObj, NGAMS_NOTIF_NO_DISKS,
"NO DISKS AVAILABLE", errMsg)
raise Exception(errMsg)
# Set the log cache to 1 during the handling of the file.
plugIn = srvObj.getMimeTypeDic()[mimeType]
logger.debug("Invoking DAPI: %s to handle file: %s", plugIn,
stagingFile)
plugInMethod = loadPlugInEntryPoint(plugIn)
resMain = plugInMethod(srvObj, reqPropsObjLoc)
# Move the file to final destination.
st = time.time()
mvFile(reqPropsObjLoc.getStagingFilename(),
resMain.getCompleteFilename())
iorate = reqPropsObjLoc.getSize() / (time.time() - st)
ngamsArchiveUtils.postFileRecepHandling(srvObj, reqPropsObjLoc,
resMain, trgDiskInfo)
except Exception as e:
# If another error occurs, than one qualifying for Back-Log
# Buffering the file, we have to log an error.
if (ngamsHighLevelLib.performBackLogBuffering(srvObj.getCfg(),
reqPropsObjLoc, e)):
logger.exception(
"Tried to archive local file %s, keeping original file",
filename)
return [NGAMS_FAILURE, str(e), NGAMS_FAILURE]
else:
logger.exception("Tried to archive local file %s, " + \
"moving it to Bad Files Directory " + \
"-- cannot be handled", filename)
ngamsHighLevelLib.moveFile2BadDir(srvObj.getCfg(), filename)
# Remove pickle file if available.
pickleObjFile = filename + "." + NGAMS_PICKLE_FILE_EXT
if (os.path.exists(pickleObjFile)):
logger.debug("Removing Back-Log Buffer Pickle File: %s",
pickleObjFile)
rmFile(pickleObjFile)
return [NGAMS_FAILURE, str(e), NGAMS_FAILURE]
# If the file was handled successfully, we remove it from the
# Back-Log Buffer Directory unless the local file was a log-file
# in which case we leave the cleanup to the Janitor-Thread.
if stagingFile.find('LOG-ROTATE') > -1:
logger.debug("Successfully archived local file: %s", filename)
else:
logger.debug(
"Successfully archived local file: %s. Removing staging file.",
filename)
rmFile(stagingFile)
rmFile(stagingFile + "." + NGAMS_PICKLE_FILE_EXT)
logger.debug("Archived local file: %s. Time (s): %.3f", filename,
time.time() - archive_start)
return (resMain, trgDiskInfo, iorate)