def getControllers(): """ Query the controllers available, and return this in a list. Returns: List with available 3ware controllers (list). """ T = TRACE() cmd = "sudo /usr/local/sbin/tw_cli info" stat, out = commands.getstatusoutput(cmd) if (stat): raise Exception, "Error invoking 3ware Command Line Tool: " + str(out) contList = [] for line in out.split("\n"): line = line.strip() if (line): if (line.find("Controller") == 0): # "Controller 1: 8506-4LP (4)" contNo = line.split(" ")[1].split(":")[0] contList.append(contNo) elif (line[0] == "c"): contNo = int(line.split(" ")[0][1:]) contList.append(contNo) contList.sort() return contList
def _getRefCounts(): """ Return information about all object allocated and the number of references pointing to each object. This can be used to check a running server for memory (object) leaks. Taken from: http://www.nightmare.com/medusa/memory-leaks.html. Returns: """ T = TRACE() d = {} sys.modules # Collect all classes for m in sys.modules.values(): for sym in dir(m): o = getattr(m, sym) if type(o) is types.ClassType: d[o] = sys.getrefcount(o) # sort by refcount pairs = map(lambda x: (x[1], x[0]), d.items()) pairs.sort() pairs.reverse() return pairs
def prepFile(reqPropsObj, parDic): """ Prepare the file. If it is compressed, decompress it into a temporary filename. reqPropsObj: NG/AMS request properties object (ngamsReqProps). parDic: Dictionary with parameters for the DAPI. This is generated with ngamsPlugInApi.parseDapiPlugInPars() (Dictionary). Returns: Tuple containing: (<DP ID>, <Date Obs. Night>, <Compr. Ext.>) (tuple). """ T = TRACE() # If the file is already compressed, we have to decompress it. comprExt = '' compression = parDic["compression"] if compression and 'gzip' in compression: comprExt = 'gz' tmpFn = reqPropsObj.getStagingFilename() if tmpFn.lower().endswith('.gz'): newFn = os.path.splitext(tmpFn)[0] logger.debug("Decompressing file using gzip: %s", tmpFn) subprocess.check_call(['gunzip', '-f', tmpFn], shell=False) logger.debug("Decompression success: %s", newFn) reqPropsObj.setStagingFilename(newFn) checkFitsFileSize(reqPropsObj.getStagingFilename()) dpIdInfo = getDpIdInfo(reqPropsObj.getStagingFilename()) return dpIdInfo[1], dpIdInfo[2], comprExt
def getFitsKeys(fitsFile, keyList): """ Get a FITS keyword from a FITS file. A dictionary is returned whereby the keys in the keyword list are the dictionary keys and the value the elements that these refer to. fitsFile: Filename of FITS file (string). keyList: Tuple of keys for which to extract values (tuple). Returns: Dictionary with the values extracted of the format: {<key 1>: [<val hdr 0>, <val hdr 1> ...], <key 2>: ...} (dictionary). """ T = TRACE() import astropy.io.fits as pyfits keyDic = defaultdict(list) try: for key in keyList: vals = pyfits.getval(fitsFile, key) if isinstance(vals, basestring): vals = [vals] keyDic[key] = list(vals) return keyDic except Exception, e: msg = ". Error: %s" % str(e) errMsg = genLog("NGAMS_ER_RETRIEVE_KEYS", [str(keyList), fitsFile + msg]) logger.exception(errMsg) raise
def ngamsExampleCacheCtrlPI(srvObj, cacheEntryObj): """ Example Cache Control Plug-in to control the cache holding based on an expiration time for each cached file. srvObj: Reference to NG/AMS Server Object (ngamsServer). cacheEntryObj: Cache Entry Object containing the information for the cached file (ngamsCacheEntry). Returns: Returns True if the file can be deleted from the cache, otherwise False (boolean). """ T = TRACE() try: global ngamsExampleCacheCtrlPI_maxCacheTime if (not ngamsExampleCacheCtrlPI_maxCacheTime): plugInPars = srvObj.getCfg().\ getVal("Caching[1].CacheControlPlugInPars") plugInParDic = ngamsPlugInApi.parseRawPlugInPars(plugInPars) try: _maxCacheTime = float(plugInParDic["max_cache_time"]) except: msg = "Missing plug-in parameter: max_cache_time" raise Exception, msg # Check if the cache time is exceeded. if ((time.time() - cacheEntryObj.getCacheTime()) > _maxCacheTime): return True return False except Exception: raise
def checkForDblExt(complFilename, relFilename): """ If if the File ID is derived from the URI, it might be that there is a double extension due to the way the ngamsPlugInApi.genFileInfo() generates the filename. This function checks double extensions and remove one of them in case there are two. complFilename: Complete filename (string). relFilename: Relative filename (string). Returns: Tuple with complete filename and relative filename (tuple/string). """ T = TRACE() filename2, ext1 = os.path.splitext(complFilename) filename3, ext2 = os.path.splitext(filename2) if ((ext1 != "") and (ext1 == ext2)): # Remove one of the extensions. complFilename = complFilename[0:-len(ext1)] relFilename = relFilename[0:-len(ext1)] return (complFilename, relFilename)
def wakeUpHost(srvObj, suspHost): """ Wake up a host which has been suspended. After invoking the appropriate Wake-Up Plug-In it is checked within the time-out defined in the NG/AMS Configuration File if the server, which was woken up is up and running. If this is not the case within the specified time-out, an exception is thrown. srvObj: Reference to instance of the NG/AMS server (ngamsServer). suspHost: Host name of the suspended host (string). Returns: Void. """ T = TRACE() wakeUpPi = srvObj.getCfg().getWakeUpPlugIn() portNo = srvObj.getDb().getPortNoFromHostId(suspHost) try: plugInMethod = loadPlugInEntryPoint(wakeUpPi) plugInMethod(srvObj, suspHost) ipAddress = srvObj.getDb().getIpFromHostId(suspHost) ngamsHighLevelLib.pingServer(ipAddress, portNo, srvObj.getCfg().getWakeUpCallTimeOut()) except Exception: logger.exception("Error waking up host %s", suspHost) raise
def _genObjectStatus(): """ Generate a report with information about objects allocated, numbers of references to each object and other information that can be used to track down memory (object) leaks in the server. Returns: Object report (string). """ T = TRACE() rep = "NG/AMS SERVER OBJECT STATUS REPORT\n\n" import gc gc.set_debug(gc.DEBUG_COLLECTABLE | gc.DEBUG_UNCOLLECTABLE | gc.DEBUG_INSTANCES | gc.DEBUG_OBJECTS) rep += "=Garbage Collector Status:\n\n" rep += "Enabled: %d\n" % gc.isenabled() rep += "Unreachable objects: %d\n" % gc.collect() rep += "Threshold: %s\n" % str(gc.get_threshold()) #rep += "Objects:\n" #rep += str(gc.get_objects()) + "\n" rep += "Garbage:\n" rep += str(gc.garbage) + "\n\n" # Dump refence count status of all objects allocated into file specified. rep += "=Object Reference Counts:\n\n" for objInfo in _genRefCountRep(): rep += "%-4d %s\n" % (objInfo[0], objInfo[1]) rep += "\n=EOF" return rep
def checkStagingAreas(srvObj): """ Check the Staging Areas of the disks registered, and in case files are found on these move htese to the Back-Log Area. srvObj: Reference to NG/AMS server class object (ngamsServer). Returns: Void. """ T = TRACE() diskList = ngamsDiskUtils.\ getDiskInfoForMountedDisks(srvObj.getDb(), srvObj.getHostId(), srvObj.getCfg().\ getRootDirectory()) # Generate first a dictionary with all files in the staging areas. stagingFileDic = {} for disk in diskList: stagingArea = disk.getStagingArea() if (stagingArea != ""): fileList = glob.glob(stagingArea + "/*") fileList += glob.glob(stagingArea + "/.NGAMS*") for filename in fileList: stagingFileDic[filename] = 1 # Go through all files in the staging file dictionary and move them to # the Bad Files Area. for filename in stagingFileDic.keys(): ngamsHighLevelLib.moveFile2BadDir(srvObj.getCfg(), filename)
def remDisk(srvObj, reqPropsObj, diskId, execute): """ Select a disk for removal and remove the information if so specified from the NGAS DB and delete all information on the disk. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). diskId: ID of disk. Complete ID must be specified. I.e., no wildcards are handled (string). execute: If set to 1 the information about the disk will be deleted. Otherwise only the information about the disk selected for deletion will be queried (integer/0|1). Returns: Status object contained information about disk selected for deletion/deleted (ngamsStatus). """ T = TRACE() tmpFilePat = ngamsHighLevelLib.genTmpFilename(srvObj.getCfg(), "REMDISK_CMD") try: status = _remDisk(srvObj, reqPropsObj, diskId, execute, tmpFilePat) return status finally: rmFile(tmpFilePat + "*")
def saveInStagingFile(ngamsCfgObj, reqPropsObj, httpRef, stagingFilename, diskInfoObj): """ Save the data ready on the HTTP channel, into the given Staging Area file. ngamsCfgObj: NG/AMS Configuration (ngamsConfig). reqPropsObj: NG/AMS Request Properties object (ngamsReqProps). stagingFilename: Staging Area Filename as generated by ngamsHighLevelLib.genStagingFilename() (string). diskInfoObj: Disk info object. Only needed if mutual exclusion is required for disk access (ngamsDiskInfo). Returns: Void. """ T = TRACE() try: blockSize = ngamsCfgObj.getBlockSize() return saveFromHttpToFile(ngamsCfgObj, reqPropsObj, httpRef, stagingFilename, blockSize, 1, diskInfoObj) except Exception as e: errMsg = genLog("NGAMS_ER_PROB_STAGING_AREA", [stagingFilename, str(e)]) logger.exception(errMsg) raise
def handleCmd(srvObj, reqPropsObj, httpRef): """ Handle CACHEDEL Command. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: Void. """ T = TRACE() diskId = None fileId = None fileVersion = None for httpPar in reqPropsObj.getHttpParNames(): if (httpPar == "disk_id"): diskId = reqPropsObj.getHttpPar("disk_id") elif (httpPar == "file_id"): fileId = reqPropsObj.getHttpPar("file_id") elif (httpPar == "file_version"): fileVersion = int(reqPropsObj.getHttpPar("file_version")) else: pass if ((not diskId) or (not fileId) or (not fileVersion)): msg = "Must specify disk_id/file_id/file_version for " +\ "CACHEDEL Command" raise Exception(msg) cacheDel(srvObj, reqPropsObj, httpRef, diskId, fileId, fileVersion)
def saveInStagingFile(srvObj, ngamsCfgObj, reqPropsObj, stagingFilename, startByte): """ Save the data ready on the HTTP channel, into the given Staging Area file. ngamsCfgObj: NG/AMS Configuration (ngamsConfig). reqPropsObj: NG/AMS Request Properties object (ngamsReqProps). stagingFilename: Staging Area Filename as generated by ngamsHighLevelLib.genStagingFilename() (string). diskInfoObj: Disk info object. Only needed if mutual exclusion is required for disk access (ngamsDiskInfo). Returns: Void. """ TRACE() blockSize = ngamsCfgObj.getBlockSize() fetchMethod = 'HTTP' if ngamsCfgObj.getVal("Mirroring[1].fetch_method"): fetchMethod = ngamsCfgObj.getVal("Mirroring[1].fetch_method") if fetchMethod == 'RSYNC': info = ngamsCmd_RSYNCFETCH.saveToFile(srvObj, ngamsCfgObj, reqPropsObj, stagingFilename, blockSize, startByte) else: info = ngamsCmd_HTTPFETCH.saveToFile(srvObj, ngamsCfgObj, reqPropsObj, stagingFilename, blockSize, startByte) return info
def _handleFileList(srvObj, reqPropsObj, httpRef): """ Handle STATUS?file_list... Command. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: The File List ID allocated for this request (string). """ T = TRACE() # Should a lower limit for the ingestion date be taken into account. fromIngDate = None if (reqPropsObj.hasHttpPar("from_ingestion_date")): tmpTromIngDate = reqPropsObj.getHttpPar("from_ingestion_date") fromIngDate = fromiso8601(tmpTromIngDate) # Handle the unique flag. If this is specified, only information for unique # File ID/Version pairs are returned. unique = False if (reqPropsObj.hasHttpPar("unique")): unique = int(reqPropsObj.getHttpPar("unique")) # Dump the file information needed. fileListId = genUniqueId() dbmBaseName = STATUS_FILE_LIST_DBM_TAG % fileListId fileInfoDbmName = ngamsHighLevelLib.genTmpFilename(srvObj.getCfg(), dbmBaseName) # Dump the file info from the DB. Deal with uniqueness quickly # (instead of using a new file like before) try: fileInfoDbm = ngamsDbm.ngamsDbm(fileInfoDbmName, 0, 1) fileCount = 1 unique_files = set() for f in srvObj.db.files_in_host(srvObj.getHostId(), from_date=fromIngDate): if unique: key = str('%s_%d' % (f[2], f[3])) if key in unique_files: continue else: key = str(fileCount) fileInfoDbm.add(key, f) fileCount += 1 except Exception as e: rmFile(fileInfoDbmName) msg = "Problem generating file list for STATUS Command. " +\ "Parameters: from_ingestion_date=%s. Error: %s" %\ (str(fromIngDate), str(e)) raise Exception(msg) return fileListId
def extractData(result, resourceId, verbose=0): """ INPUT: result: list of strings containing multipart messages RETURNS: xyData: dictionary, """ T = TRACE(3) if (verbose >= 3): print "Entering extractData with parameters " + \ "type(result): %s, resourceId: %s" % (str(type(result)), resourceId) eP = MultipartHandler.Parser() xData = [] yData = [] xyData = {} if len(result) == 0: errMsg = "[ERROR]: No data returned. Check whether resourceId is correct: %s" % resourceId raise Exception(errMsg) resultClean = [] for res in result: if len(res) != 0: resultClean.append(res) else: logger.warning("Empty document detected and ignored!") for res in resultClean: try: msg = eP.parsestr(res) # and parse the string except Exception: logger.exception("email parsing failed") raise # get the first xml part of the email... xmlParts = MultipartHandler.getMimeParts(msg, 'text/xml') if len(xmlParts) == 0: errMsg = "MonitorDataCli.extractData: No text/xml part found!" raise Exception(errMsg) else: xml = xmlParts[0].get_payload() # There should only be one! # ...and parse it try: root = MultipartHandler.VOTable.parseString(xml) except Exception, e: # print xml errMsg = "MonitorDataCli.extractData: XML parsing failed: %s " % str( e) raise Exception(errMsg) try: (res, cidI, cids) = MultipartHandler.interpretVotable(root, selection=resourceId, verbose=verbose) except Exception, e: errMsg = "ERROR interpreting VOTABLE: %s" % str(e) raise Exception(errMsg)
def locateArchiveFile(srvObj, fileId, fileVersion = -1, diskId = "", hostId = "", reqPropsObj = None): """ Locate the file indicated by the File ID. Returns a list containing the necessary information for retrieving the file: [<Location>, <File Host>, <IP Address>, <Port No>, <Mount Point>, <Filename>, <File ID>, <File Version>, <Mime-Type>] - whereby: <Location> = Location of the file (NGAMS_HOST_LOCAL, NGAMS_HOST_CLUSTER, NGAMS_HOST_DOMAIN, NGAMS_HOST_REMOTE). <File Host> = Host ID of host to be contacted to get access to the file. <IP Address> = IP Address of host to be contacted to get access to the file. <Port No> = Port number used by the NG/AMS Server. <Mount Point> = Mount point at which the file is residing. <Filename> = Name of file relative to mount point. <File ID> = ID of file. <File Version> = Version of file. <Mime-Type> = Mime-type of file (as registered in NGAS). srvObj: Reference to NG/AMS server class object (ngamsServer). fileId: File ID of file to locate (string). fileVersion: Version of the file (integer). diskId: ID of the disk hosting the file (string). hostId: ID of the host where the file is located (string). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps|None). Returns: List with information about file location (list). """ T = TRACE() # Get a list with the candidate files matching the query conditions. res = srvObj.getDb().getFileInfoFromFileId(fileId, fileVersion, diskId, ignore=0, dbCursor=False) # r[-2] is the host_id, r[-1] is the mount point all_info = [] for r in res: file_info = ngamsFileInfo.ngamsFileInfo().unpackSqlResult(r) all_info.append((file_info, r[-2], r[-1])) return _locateArchiveFile(srvObj, fileId, fileVersion, diskId, hostId, reqPropsObj, all_info)
def ngamsZHLDapi(srvObj, reqPropsObj): """ Generic Data Archiving Plug-In to handle archiving of any file. srvObj: Reference to NG/AMS Server Object (ngamsServer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). Returns: Standard NG/AMS Data Archiving Plug-In Status as generated by: ngamsPlugInApi.genDapiSuccessStat() (ngamsDapiStatus). """ T = TRACE() # For now the exception handling is pretty basic: # If something goes wrong during the handling it is tried to # move the temporary file to the Bad Files Area of the disk. logger.debug("Plug-In handling data for file: %s", os.path.basename(reqPropsObj.getFileUri())) try: parDic = {} handlePars(reqPropsObj, parDic) diskInfo = reqPropsObj.getTargDiskInfo() stgFile = reqPropsObj.getStagingFilename() ext = os.path.splitext(stgFile)[1][1:] # Generate file information. logger.debug("Generate file information") dateDir = toiso8601(fmt=FMT_DATE_ONLY) fileVersion, relPath, relFilename,\ complFilename, fileExists =\ ngamsPlugInApi.genFileInfo(srvObj.getDb(), srvObj.getCfg(), reqPropsObj, diskInfo, reqPropsObj.\ getStagingFilename(), parDic[FILE_ID], parDic[FILE_ID], [dateDir]) complFilename, relFilename = checkForDblExt(complFilename, relFilename) # Compress the file if requested. uncomprSize, archFileSize, format, compression, comprExt =\ compressFile(srvObj, reqPropsObj, parDic) if (comprExt != ""): complFilename += ".%s" % comprExt relFilename += ".%s" % comprExt logger.debug( "DAPI finished processing file - returning to host application") insertFitsRecords(srvObj, reqPropsObj, stgFile) return ngamsPlugInApi.genDapiSuccessStat( diskInfo.getDiskId(), relFilename, parDic[FILE_ID], fileVersion, format, archFileSize, uncomprSize, compression, relPath, diskInfo.getSlotId(), fileExists, complFilename) except Exception: logger.exception("Error occurred in DAPI") raise
def checkSpuriousFiles(srvObj, tmpFilePat=None, hostId=None, diskId=None, fileId=None, fileVersion=None): """ Check if there are any spurious files in the DB either marked as to be ignored or having a status indicating a problem. If such are found according to the criterias defined, these are added in a DBM DB, which name is returned. srvObj: Reference to NG/AMS server class object (ngamsServer). tmpFilePat: Pattern to apply for temporary files (string). hostId: Name of NGAS host on which the files reside (string). diskId: Disk ID of disk to take into account (string|None). fileId: File ID of file(s) to take into account (string|None). fileVersion: Version of file(s) to take into account (integer|None). Returns: Returns name of DBM DB with references to spurious files found (string). """ T = TRACE() if (hostId == ""): hostId = None if (diskId == ""): diskId = None if (fileId == ""): fileId = None if (fileVersion == -1): fileVersion = None # DBM DB containing information about spurious files. filename = "_SPURIOUS_FILES" if (tmpFilePat): spuriousFilesDbmName = os.path.normpath(tmpFilePat + filename) else: spuriousFilesDbmName = ngamsHighLevelLib.\ genTmpFilename(srvObj.getCfg(), filename) spuriousFilesDbm = ngamsDbm.ngamsDbm(spuriousFilesDbmName, writePerm=1) # Check that there are no spurious files in connection with this disk in # the DB (where ngas_files.file_ignore != 0 or ngas_files.status != "1*******" files = srvObj.db.getFileSummarySpuriousFiles1(hostId, diskId, fileId, fileVersion, fetch_size=200) for fileInfo in files: spuriousFilesDbm.addIncKey(fileInfo) spuriousFilesDbm.sync() del spuriousFilesDbm return spuriousFilesDbmName
def _checkFileAccess(srvObj, reqPropsObj, httpRef, fileId, fileVersion=-1, diskId=""): """ Check if a file is accessible either on the local host or on a remotely located host. srvObj: Instance of the server object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). fileId: File ID (string). fileVersion: File Version (integer). diskId: Disk ID (string). Returns: Returns message indicating if the file is available (string). """ T = TRACE() logger.debug("Checking for access to file with ID: %s", fileId) # Check if the file is located on this host, or if the request should be # forwarded (if this server should act as proxy). location, fileHost, ipAddress, filePortNo, mountPoint, filename, fileId,\ fileVersion, mimeType =\ ngamsFileUtils.locateArchiveFile(srvObj, fileId, fileVersion, diskId) if (location != NGAMS_HOST_LOCAL): # Go and get it! host, port = srvObj.get_remote_server_endpoint(fileHost) pars = (('file_access', fileId), ('file_version', fileVersion), ('disk_id', diskId)) resp = ngamsHttpUtils.httpGet(host, port, 'STATUS', pars, timeout=60) with contextlib.closing(resp): return ngamsStatus.to_status(resp, fileHost, 'STATUS').getMessage() else: # First check if this system allows for Retrieve Requests. if (not srvObj.getCfg().getAllowRetrieveReq()): msg = genLog("NGAMS_INFO_SERVICE_UNAVAIL", ["File Retrieval"]) else: fileHost = "%s:%d" % (getHostName(), filePortNo) msg = genLog("NGAMS_INFO_FILE_AVAIL", [fileId + "/Version: " + str(fileVersion), fileHost]) return msg
def handlePars(reqPropsObj, parDic): """ Parse/handle the HTTP parameters. reqPropsObj: NG/AMS request properties object (ngamsReqProps). parDic: Dictionary with the parameters (dictionary). Returns: Void. """ T = TRACE() # Get parameters. logger.debug("Get request parameters") parDic[TARG_MIME_TYPE] = None parDic[FILE_ID] = None parDic[VERSIONING] = 1 parDic[COMPRESSION] = None parDic[COMPRESSION_EXT] = None if (reqPropsObj.hasHttpPar(TARG_MIME_TYPE)): parDic[TARG_MIME_TYPE] = reqPropsObj.getHttpPar(TARG_MIME_TYPE) # If the file_id is not given, we derive it from the name of the URI. if (reqPropsObj.hasHttpPar(FILE_ID)): parDic[FILE_ID] = reqPropsObj.getHttpPar(FILE_ID) if (not parDic[FILE_ID]): if (reqPropsObj.getFileUri().find("file_id=") > 0): file_id = reqPropsObj.getFileUri().split("file_id=")[1] parDic[FILE_ID] = file_id logger.info("No file_id given, but found one in the URI: %s", parDic[FILE_ID]) else: parDic[FILE_ID] = os.path.basename(reqPropsObj.getFileUri()) logger.info("No file_id given, using basename of URI: %s", parDic[FILE_ID]) if (reqPropsObj.hasHttpPar(VERSIONING)): parDic[VERSIONING] = int(reqPropsObj.getHttpPar(VERSIONING)) # Set also the no_versioning parameter for backwards compatibility reasons if (parDic[VERSIONING]): reqPropsObj.addHttpPar("no_versioning", "0") else: reqPropsObj.addHttpPar("no_versioning", "1") if (reqPropsObj.hasHttpPar(COMPRESSION)): parDic[COMPRESSION] = reqPropsObj.getHttpPar(COMPRESSION) if (reqPropsObj.hasHttpPar(COMPRESSION_EXT)): parDic[COMPRESSION_EXT] = reqPropsObj.getHttpPar(COMPRESSION_EXT) if ((parDic[COMPRESSION] and (parDic[COMPRESSION_EXT] == None)) or (not parDic[COMPRESSION] and (parDic[COMPRESSION_EXT] != None))): raise Exception( genLog("NGAMS_ER_DAPI", [ "Parameters compression and compression_ext" "must be given together." ]))
def ngamsSuspensionPlugIn(srvObj): """ Suspension Plug-In to suspend an NGAS host. srvObj: Reference to instance of the NG/AMS Server (ngamsServer). Returns: Void. """ T = TRACE() commands.getstatusoutput("sudo /sbin/shutdown -h now")
def handleCmd(srvObj, reqPropsObj, httpRef): """ Handle the RUN TASK (RUNTASK) Command. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: Void. """ T = TRACE() global queScanThread httpMethod = reqPropsObj.getHttpMethod() if (httpMethod != 'POST'): errMsg = 'OK' if (reqPropsObj.hasHttpPar('action')): action_req = reqPropsObj.getHttpPar('action') if ('cancel' == action_req): if (reqPropsObj.hasHttpPar('task_id')): taskId = reqPropsObj.getHttpPar('task_id') cancelDict[taskId] = 1 if (g_mrLocalTask and taskId == g_mrLocalTask._taskId): res = g_mrLocalTask.stop() if (res[0]): errMsg = 'Cancel result failed: %s\n' % str(res[1]) else: errMsg = 'task_id is missing' else: errMsg = 'Unknown RUNTASK command action %s' % action_req else: errMsg = 'RUNTASK command needs action for GET request\n' httpRef.send_data(errMsg, NGAMS_TEXT_MT) else: postContent = _getPostContent(srvObj, reqPropsObj, httpRef) mrLocalTask = pickle.loads(postContent) if (not mrLocalTask): errMsg = 'Cannot instantiate local task from POST' mrr = MRLocalTaskResult(None, -2, errMsg) httpRef.send_data(pickle.dumps(mrr), NGAMS_TEXT_MT) else: logger.debug('Local task %s is submitted', mrLocalTask._taskId) mrr = MRLocalTaskResult(mrLocalTask._taskId, 0, '') httpRef.send_data(pickle.dumps(mrr), NGAMS_TEXT_MT) args = (srvObj, mrLocalTask) scheduleThread = threading.Thread(None, _scheduleQScanThread, 'SCHEDULE_THRD', args) scheduleThread.setDaemon(0) scheduleThread.start()
def ngamsGenericOfflinePlugIn(srvObj, reqPropsObj=None): """ Generic NGAS Offline Plug-In. srvObj: Reference to instance of the NG/AMS Server class (ngamsServer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). Returns: Void. """ T = TRACE()
def ngamsTestWakeUpPlugIn(srvObj, hostId): """ Dummy Wake-Up Plug-In to test the handling of the NGAS host suspension. srvObj: Reference to instance of the NG/AMS Server (ngamsServer). hostId: Name of NGAS host to be woken up (string). Returns: Void. """ T = TRACE(2) srvObj.getDb().resetWakeUpCall(hostId, 1)
def __handleCmd(srvObj, reqPropsObj): """ Handle the Mirroring Archive (MIRRARCHIVE) Command. srvObj: Reference to NG/AMS server class object (ngamsServer).setState reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). Returns: Void. """ TRACE() # Is this NG/AMS permitted to handle Archive Requests? logger.debug( "Checking if this NG/AMS permitted to handle Archive Requests?") if (not srvObj.getCfg().getAllowArchiveReq()): errMsg = genLog("NGAMS_ER_ILL_REQ", ["Archive"]) raise Exception, errMsg # Generate staging filename. stgFilename = reqPropsObj.getStagingFilename() logger.info("staging filename is: %s", stgFilename) startByte = 0 if (os.path.exists(stgFilename) == 0): logger.debug('this is a new staging file') else: startByte = os.path.getsize(stgFilename) logger.debug( 'staging file already exists, requesting resumption of download from byte %d', startByte) # Set reference in request handle object to the read socket. try: # Retrieve file_id and file_version from request proposal file_id = reqPropsObj.fileinfo['fileId'] file_version = reqPropsObj.fileinfo['fileVersion'] logger.debug("Got file_id=%s and file_version=%s", file_id, file_version) # Retrieve file contents (from URL, archive pull, or by storing the body # of the HTTP request, archive push). logger.info("Saving in staging file: %s", stgFilename) stagingInfo = saveInStagingFile(srvObj, srvObj.getCfg(), reqPropsObj, stgFilename, startByte) reqPropsObj.incIoTime(stagingInfo[0]) checksumPlugIn = "ngamsGenCrc32" checksum = stagingInfo[1] except ngamsFailedDownloadException.FailedDownloadException, e: raise
def handlePars(reqPropsObj, parDic): """ Parse/handle the HTTP parameters. reqPropsObj: NG/AMS request properties object (ngamsReqProps). parDic: Dictionary with the parameters (dictionary). Returns: Void. """ T = TRACE() # Get parameters. logger.debug("Get request parameters") parDic[TARG_MIME_TYPE] = None parDic[FILE_ID] = None parDic[VERSIONING] = 1 if (reqPropsObj.hasHttpPar(TARG_MIME_TYPE)): parDic[TARG_MIME_TYPE] = reqPropsObj.getHttpPar(TARG_MIME_TYPE) # If the file_id is not given, we derive it from the name of the URI. if (reqPropsObj.hasHttpPar(FILE_ID)): parDic[FILE_ID] = reqPropsObj.getHttpPar(FILE_ID) if (not parDic[FILE_ID]): if (reqPropsObj.getFileUri().find("file_id=") > 0): file_id = reqPropsObj.getFileUri().split("file_id=")[1] parDic[FILE_ID] = os.path.basename(file_id) logger.info("No file_id given, but found one in the URI: %s", parDic[FILE_ID]) else: parDic[FILE_ID] = os.path.basename(reqPropsObj.getFileUri()) logger.info("No file_id given, using basename of fileUri: %s", parDic[FILE_ID]) # TODO: It would seem like 'versioning' is not used anywhere, # but still gets more priority than 'no_versioning'. # which is used everywhere if VERSIONING in reqPropsObj: parDic[VERSIONING] = int(reqPropsObj[VERSIONING]) elif 'no_versioning' in reqPropsObj: parDic[VERSIONING] = 0 if int(reqPropsObj['no_versioning']) else 1 # Set also the no_versioning parameter for backwards compatibility reasons if (parDic[VERSIONING]): reqPropsObj.addHttpPar("no_versioning", "0") else: reqPropsObj.addHttpPar("no_versioning", "1") extract_compression_params(reqPropsObj, parDic)
def ngamsDiskSyncPlugIn(srvObj): """ Disk Sync Plug-In to flush the cache of the 3ware controller. srvObj: Reference to instance of NG/AMS Server class (ngamsServer). Returns: Void. """ T = TRACE() # Sync filesystem to ensure file received on disk. logger.debug("Performing OS sync command ...") with _proc_startup_lock: subprocess.call("sync")
def rescanCont(contId): """ Rescan a 3ware controller. contId: ID (number) of the controller (integer). Returns: Void. """ return # Don't want to execute this T = TRACE() cmd = "sudo /usr/local/sbin/tw_cli /c%d rescan" % (int(contId)) logger.debug("Invoking command to rescan 3ware unit: %s ...", cmd) stat, out = commands.getstatusoutput(cmd) logger.debug("Result of command: %s to rescan 3ware unit: %d", cmd, stat)
def handleCmd(srvObj, reqPropsObj, httpRef): """ Handle CLONE command. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: Void. """ T = TRACE() # Is this NG/AMS permitted to handle Archive Requests? if (not srvObj.getCfg().getAllowArchiveReq()): errMsg = genLog("NGAMS_ER_ILL_REQ", ["Clone"]) raise Exception(errMsg) # Check if State/Sub-State correct for perfoming the cloning. srvObj.checkSetState("Command CLONE", [NGAMS_ONLINE_STATE], [NGAMS_IDLE_SUBSTATE, NGAMS_BUSY_SUBSTATE]) # Get the parameters from the query. if (reqPropsObj.hasHttpPar("file_id")): fileId = reqPropsObj.getHttpPar("file_id") else: fileId ="" if (reqPropsObj.hasHttpPar("disk_id")): diskId = reqPropsObj.getHttpPar("disk_id") else: diskId = "" if (reqPropsObj.hasHttpPar("file_version")): fileVersion = int(reqPropsObj.getHttpPar("file_version")) else: fileVersion = -1 if (reqPropsObj.hasHttpPar("target_disk_id")): targetDiskId = reqPropsObj.getHttpPar("target_disk_id") else: targetDiskId = "" # Carry out the cloning. clone(srvObj, diskId, fileId, fileVersion,targetDiskId,reqPropsObj,httpRef)
def addSubscriber(srvObj, subscrObj): """ Add a Subscriber to the list of Subscribers. The information about the Subscriber is also updated in the DB. srvObj: Reference to NG/AMS Server object (ngamsServer). subscrObj: Subscriber Object (ngamsSubscriber). Returns: Void. """ T = TRACE() srvObj.getDb().insertSubscriberEntry(subscrObj) #subscrObj.write(srvObj.getDb()) srvObj.registerSubscriber(subscrObj)