def checkStagingAreas(srvObj): """ Check the Staging Areas of the disks registered, and in case files are found on these move htese to the Back-Log Area. srvObj: Reference to NG/AMS server class object (ngamsServer). Returns: Void. """ T = TRACE() diskList = ngamsDiskUtils.\ getDiskInfoForMountedDisks(srvObj.getDb(), srvObj.getHostId(), srvObj.getCfg().\ getRootDirectory()) # Generate first a dictionary with all files in the staging areas. stagingFileDic = {} for disk in diskList: stagingArea = disk.getStagingArea() if (stagingArea != ""): fileList = glob.glob(stagingArea + "/*") fileList += glob.glob(stagingArea + "/.NGAMS*") for filename in fileList: stagingFileDic[filename] = 1 # Go through all files in the staging file dictionary and move them to # the Bad Files Area. for filename in stagingFileDic.keys(): ngamsHighLevelLib.moveFile2BadDir(srvObj.getCfg(), filename)
def checkStagingAreas(srvObj): """ Check the Staging Areas of the disks registered, and in case files are found on these move htese to the Back-Log Area. srvObj: Reference to NG/AMS server class object (ngamsServer). Returns: Void. """ diskList = ngamsDiskUtils.\ getDiskInfoForMountedDisks(srvObj.getDb(), srvObj.getHostId(), srvObj.getCfg().\ getRootDirectory()) for disk in diskList: stagingArea = disk.getStagingArea() if stagingArea: files_to_move = glob.glob(stagingArea + "/*") files_to_move += glob.glob(stagingArea + "/.NGAMS*") for f in files_to_move: ngamsHighLevelLib.moveFile2BadDir(disk, f)
ngamsArchiveUtils.postFileRecepHandling(srvObj, reqPropsObjLoc, resMain, trgDiskInfo) except Exception, e: # If another error occurs, than one qualifying for Back-Log # Buffering the file, we have to log an error. if (ngamsHighLevelLib.performBackLogBuffering(srvObj.getCfg(), reqPropsObjLoc, e)): logger.exception( "Tried to archive local file %s, keeping original file", filename) return [NGAMS_FAILURE, str(e), NGAMS_FAILURE] else: logger.exception("Tried to archive local file %s, " + \ "moving it to Bad Files Directory " + \ "-- cannot be handled", filename) ngamsHighLevelLib.moveFile2BadDir(srvObj.getCfg(), filename) # Remove pickle file if available. pickleObjFile = filename + "." + NGAMS_PICKLE_FILE_EXT if (os.path.exists(pickleObjFile)): logger.debug("Removing Back-Log Buffer Pickle File: %s", pickleObjFile) rmFile(pickleObjFile) return [NGAMS_FAILURE, str(e), NGAMS_FAILURE] # If the file was handled successfully, we remove it from the # Back-Log Buffer Directory unless the local file was a log-file # in which case we leave the cleanup to the Janitor-Thread. if stagingFile.find('LOG-ROTATE') > -1: logger.debug("Successfully archived local file: %s", filename) else: logger.debug(
def archiveFromFile(srvObj, filename, noReplication=0, mimeType=None, reqPropsObj=None): """ Archive a file directly from a file as source. srvObj: Reference to NG/AMS Server Object (ngamsServer). filename: Name of file to archive (string). noReplication: Flag to enable/disable replication (integer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). Returns: Execution result object of DAPI """ T = TRACE() logger.debug("Archiving file: %s", filename) logger.debug("Mimetype used is %s", mimeType) if (reqPropsObj): logger.debug("Request Properties Object given - using this") reqPropsObjLoc = reqPropsObj else: logger.debug("No Request Properties Object given - creating one") reqPropsObjLoc = ngamsArchiveUtils.ngamsReqProps.ngamsReqProps() stagingFile = filename try: if (mimeType == None): mimeType = ngamsHighLevelLib.determineMimeType( srvObj.getCfg(), filename) archive_start = time.time() # Prepare dummy ngamsReqProps object (if an object was not given). if (not reqPropsObj): reqPropsObjLoc.setMimeType(mimeType) reqPropsObjLoc.setStagingFilename(filename) reqPropsObjLoc.setHttpMethod(NGAMS_HTTP_GET) reqPropsObjLoc.setCmd(NGAMS_ARCHIVE_CMD) reqPropsObjLoc.setSize(os.path.getsize(filename)) reqPropsObjLoc.setFileUri(NGAMS_HTTP_FILE_URL + filename) reqPropsObjLoc.setNoReplication(noReplication) # If no target disk is defined, find one suitable disk. if (not reqPropsObjLoc.getTargDiskInfo()): try: trgDiskInfo = ngamsArchiveUtils.ngamsDiskUtils.\ findTargetDisk(srvObj.getHostId(), srvObj.getDb(), srvObj.getCfg(), mimeType, 0, reqSpace=reqPropsObjLoc.getSize()) reqPropsObjLoc.setTargDiskInfo(trgDiskInfo) # copy the file to the staging area of the target disk stagingFile = trgDiskInfo.getMountPoint( ) + '/staging/' + os.path.basename(filename) cpFile(filename, stagingFile) reqPropsObjLoc.setStagingFilename(stagingFile) except Exception as e: errMsg = str(e) + ". Attempting to archive local file: " +\ filename ngamsPlugInApi.notify(srvObj, NGAMS_NOTIF_NO_DISKS, "NO DISKS AVAILABLE", errMsg) raise Exception(errMsg) # Set the log cache to 1 during the handling of the file. plugIn = srvObj.getMimeTypeDic()[mimeType] logger.debug("Invoking DAPI: %s to handle file: %s", plugIn, stagingFile) plugInMethod = loadPlugInEntryPoint(plugIn) resMain = plugInMethod(srvObj, reqPropsObjLoc) # Move the file to final destination. st = time.time() mvFile(reqPropsObjLoc.getStagingFilename(), resMain.getCompleteFilename()) iorate = reqPropsObjLoc.getSize() / (time.time() - st) ngamsArchiveUtils.postFileRecepHandling(srvObj, reqPropsObjLoc, resMain, trgDiskInfo) except Exception as e: # If another error occurs, than one qualifying for Back-Log # Buffering the file, we have to log an error. if (ngamsHighLevelLib.performBackLogBuffering(srvObj.getCfg(), reqPropsObjLoc, e)): logger.exception( "Tried to archive local file %s, keeping original file", filename) return [NGAMS_FAILURE, str(e), NGAMS_FAILURE] else: logger.exception("Tried to archive local file %s, " + \ "moving it to Bad Files Directory " + \ "-- cannot be handled", filename) ngamsHighLevelLib.moveFile2BadDir(srvObj.getCfg(), filename) # Remove pickle file if available. pickleObjFile = filename + "." + NGAMS_PICKLE_FILE_EXT if (os.path.exists(pickleObjFile)): logger.debug("Removing Back-Log Buffer Pickle File: %s", pickleObjFile) rmFile(pickleObjFile) return [NGAMS_FAILURE, str(e), NGAMS_FAILURE] # If the file was handled successfully, we remove it from the # Back-Log Buffer Directory unless the local file was a log-file # in which case we leave the cleanup to the Janitor-Thread. if stagingFile.find('LOG-ROTATE') > -1: logger.debug("Successfully archived local file: %s", filename) else: logger.debug( "Successfully archived local file: %s. Removing staging file.", filename) rmFile(stagingFile) rmFile(stagingFile + "." + NGAMS_PICKLE_FILE_EXT) logger.debug("Archived local file: %s. Time (s): %.3f", filename, time.time() - archive_start) return (resMain, trgDiskInfo, iorate)