def _restoreSubscriptionInfoFromDisk(srvObj): ngas_root_dir = srvObj.getCfg().getRootDirectory() myDir = ngas_root_dir + "/SubscriptionInfo" if (not os.path.exists(myDir)): return saveFile = myDir + "/SubscriptionInfoObj" if (not os.path.exists(saveFile)): return logger.debug('Restoring subscription info from disks ...') saveObj = None try: pkl_file = open(saveFile, 'rb') saveObj = pickle.load(pkl_file) pkl_file.close() except Exception: logger.exception('Fail to restore subscription info from disks') if (saveObj == None or len(saveObj) == 0): return logger.debug( 'Appending subscription info to info list kept internally ...') try: srvObj._subscriptionSem.acquire() srvObj._subscriptionFileList += saveObj cmd = "rm " + saveFile ngamsPlugInApi.execCmd(cmd, -1) except Exception: logger.exception('Fail to append filelist to subscription info list') finally: srvObj._subscriptionSem.release()
def stageFiles(filenameList): """ Stage a list of files. The system will sort files in the order that they are archived on the tape volumes for better performance """ cmd1 = "stage -r" cmd2 = "stage -r -w" num_staged = 0 for filename in filenameList: if (isFileOnTape(filename) == 1): cmd1 = cmd1 + " " + filename cmd2 = cmd2 + " " + filename num_staged = num_staged + 1 t = ngamsPlugInApi.execCmd(cmd1, -1) exitCode = t[0] if (exitCode != 0): logger.error("Staging problem: %s, cmd: %s", str(exitCode), cmd1) return -1 t = ngamsPlugInApi.execCmd(cmd2, -1) exitCode = t[0] if (exitCode != 0): logger.error("Staging problem: %s, cmd: %s", str(exitCode), cmd2) return -1 return num_staged
def checkFitsChecksum(reqPropsObj, stgFile=None): """ Carry out a check of the DATASUM and CHECKSUM for each HDU in the file. reqPropsObj: NG/AMS request properties object (ngamsReqProps). stgFile: If specified this is taken rather than from the Request Properties Object (ngamsReqProps). Returns: Void. """ if (not stgFile): stgFile = reqPropsObj.getStagingFilename() cmd = "chksumVerFitsChksum %s" % stgFile stat, out = ngamsPlugInApi.execCmd(cmd) if (out.find("Status: OK") == -1): if (out.find("Status: NOT OK") != -1): errMsg = genLog( "NGAMS_ER_DAPI_BAD_FILE", [stgFile, "checkFitsChecksum", "Illegal CHECKSUM/DATASUM"]) else: errMsg = "Problem found while checking file: %s. " %\ out.replace("\n", " ") # Backwards compatibility with old ESO FITS CHECKSUM scheme: If the # file pass the previous scheme, we consider it as OK and do not # return the failure. cmd = "chksumGenChecksum %s" % stgFile stat, out = ngamsPlugInApi.execCmd(cmd) if (out.strip().find("0/0000000000000000") == -1): raise Exception, errMsg chksumUtil = "chksumGenChecksum" else: chksumUtil = "chksumVerFitsChksum" logger.debug("File: %s checked with: %s. Result: OK", stgFile, chksumUtil)
def execCmd(cmd, failonerror=True, okErr=[]): re = ngamsPlugInApi.execCmd(cmd) if (re[0] != 0 and not (re[0] in okErr)): errMsg = 'Fail to execute command: "%s". Exception: %s' % (cmd, re[1]) if (failonerror): raise Exception(errMsg) else: print(errMsg) return re
def execCmd(cmd, timeout): logger.debug('Executing command: %s', cmd) try: ret = ngamsPlugInApi.execCmd(cmd, timeout) except Exception, ex: if (str(ex).find('timed out') != -1): return (-1, 'Timed out (%d seconds): %s' % (timeout, cmd)) else: return (-1, str(ex))
def _saveSubscriptionInfoToDisk(srvObj): """ Before ngas server is shutdown, write files from to-be-delivered-list to disks otherwise, this information will be lost when server is restarted again """ # get the files currently being transferred but not yet completed # save them in the subscriptionInfo so that they can be resumed when server starts up li = [] idx_fileId = ngamsSubscriptionThread.FILE_ID idx_filever = ngamsSubscriptionThread.FILE_VER for k, fileInfo in srvObj._subscrDeliveryFileDic.items(): if (fileInfo == None): continue fileInfo = ngamsSubscriptionThread._convertFileInfo(fileInfo) li += [(fileInfo[idx_fileId], fileInfo[idx_filever])] logger.debug('%s is added to the subscriptionInfoList', fileInfo[idx_fileId]) # also add to files that have not yet been in the queue srvObj._subscriptionSem.acquire() srvObj._subscriptionFileList += li if (len(srvObj._subscriptionFileList) == 0): srvObj._subscriptionSem.release() logger.debug("**** subscription list is empty!!") return logger.debug("Saving subscription info to disks ...") ngas_root_dir = srvObj.getCfg().getRootDirectory() myDir = ngas_root_dir + "/SubscriptionInfo" saveFile = myDir + "/SubscriptionInfoObj" try: if (os.path.exists(saveFile)): cmd = "rm " + saveFile ngamsPlugInApi.execCmd(cmd, -1) if (not os.path.exists(myDir)): os.makedirs(myDir) output = open(saveFile, 'wb') pickle.dump(srvObj._subscriptionFileList, output) output.close() except Exception: logger.exception('Fail to save subscription info to disks') finally: srvObj._subscriptionSem.release()
def stopAsyncQService(srvObj, reqPropsObj): """ # when the server is shutdown, this is called to stop threads, and save uncompleted list back to database """ ngas_root_dir = srvObj.getCfg().getRootDirectory() myDir = ngas_root_dir + "/AsyncQService" saveFile = myDir + "/AsyncRetrieveListObj" uuids = asyncReqDic.keys() if (len(uuids) == 0): if (os.path.exists(saveFile)): cmd = "rm " + saveFile ngamsPlugInApi.execCmd(cmd, -1) return "ok after deleting the file" for sessionId in uuids: suspendHandler(srvObj, reqPropsObj, sessionId) #info(3, "Stopping - root dir = %s" % ngas_root_dir) if (not os.path.exists(myDir)): os.makedirs(myDir) """ asyncReqDic = {} #key - uuid, value - AsyncListRetrieveRequest (need to remember the original request in case of cancel/suspend/resume or server shutting down) statusResDic = {} #key - uuid, value - AsyncListRetrieveStatusResponse nextFileDic = {} #key - uuid, value - next file_id to be delivered threadDic = {} #key - uuid, value - the threadref threadRunDic = {} #key - uuid, value - 1/0, 1: run 0: stop """ saveObj = [asyncReqDic, statusResDic, nextFileDic] try: output = open(saveFile, 'wb') pickle.dump(saveObj, output) output.close() except Exception as e: ex = str(e) return ex return "ok"
def ngamsMWACortexStageDppi(srvObj, reqPropsObj, filename): """ This plugin stages the file from Tape to the disk, works ONLY in the iVEC Cortex environment. srvObj: Reference to instance of the NG/AMS Server class (ngamsServer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). filename: Name of file to process (string). Returns: DPPI return status object (ngamsDppiStatus). """ #mimeType = ngamsPlugInApi.determineMimeType(srvObj.getCfg(), filename) mimeType = "application/octet-stream" #hardcode this since our file extension is actually ".fits" resultObj = ngamsDppiStatus.ngamsDppiResult(NGAMS_PROC_FILE, mimeType, filename, filename) statusObj = ngamsDppiStatus.ngamsDppiStatus().addResult(resultObj) #procFilename, procDir = ngamsPlugInApi.prepProcFile(srvObj.getCfg(), filename) cmd = "sls -D " + filename t = ngamsPlugInApi.execCmd(cmd, -1) exitCode = t[0] if (exitCode != 0 or len(t) != 2): logger.error("Fail to query the online/offline status for file %s", filename) return statusObj #the sls -D command failed to execute, but retrieval might still go on, so just simply return empty result offline = t[1].find('offline;') if (offline != -1): # the file is offline, i.e. it is on tape logger.debug("File %s is currently on tapes, staging it for retrieval...", filename) cmd = "stage -w " + filename t = ngamsPlugInApi.execCmd(cmd, -1) #stage it back to disk cache logger.debug("File %s staging completed.", filename) return statusObj
def ngamsMWA_Compress_FilterPlugin(srvObj, plugInPars, filename, fileId, fileVersion=-1, reqPropsObj=None): """ srvObj: Reference to NG/AMS Server Object (ngamsServer). plugInPars: Parameters to take into account for the plug-in execution (string). fileId: File ID for file to test (string). filename: Filename of (complete) (string). fileVersion: Version of file to test (integer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). Returns: 0 if the file does not match, 1 if it matches the conditions (integer/0|1). """ if (not isMWAVisFile(fileId)): return 0 # only add MWA Vis FITS file cmd = 'head -c %d %s' % (1024 * 3, filename) try: re = ngamsPlugInApi.execCmd(cmd) except Exception as ex: if (str(ex).find('timed out') != -1): logger.error('Timed out when checking FITS header %s', cmd) else: logger.error('Exception when checking FITS header %s: %s', cmd, str(ex)) return 0 if (0 == re[0]): a = re[1].find("XTENSION= 'BINTABLE'") if (a > -1): return 0 # if the file is already compressed, do not add again else: logger.info("File %s added", filename) return 1 else: logger.warning('Fail to check header for file %s: %s', filename, re[1]) return 0
def isFileOnTape(filename): """ return 1 - on tape, 0 - not on tape, -1 - query error """ cmd = "sls -D " + filename t = ngamsPlugInApi.execCmd(cmd, -1) exitCode = t[0] if (exitCode != 0 or len(t) != 2): logger.error("Fail to query the online/offline status for file %s", filename) #print errMsg return -1 #raise error offline = t[1].find( 'offline;' ) # Do not use "archdone" any more given this reason: http://www.mail-archive.com/[email protected]/msg00419.html if (offline != -1): # the file is offline, i.e. it is on tape return 1 else: return 0
def stageFile(filename): cmd = "stage -w " + filename print("File %s is on tape, staging it now..." % filename) ngamsPlugInApi.execCmd(cmd, -1) #stage it back to disk cache print("File " + filename + " staging completed.")
def compressFile(srvObj, reqPropsObj, parDic): """ Compress the file if required. srvObj: Reference to NG/AMS Server Object (ngamsServer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). parDic: Dictionary with parameters for the DAPI. This is generated with ngamsPlugInApi.parseDapiPlugInPars() (Dictionary). Returns: Tupe containing uncompressed filesize, archived filesize and the format (mime-type) of the resulting data file and the compression method (NONE if the file is not compressed), finally, the extension added by the compression if any (tuple). """ stFn = reqPropsObj.getStagingFilename() # If a compression application is specified, apply this. uncomprSize = ngamsPlugInApi.getFileSize(stFn) comprExt = "" if (parDic[COMPRESSION]): logger.debug("Compressing file using: %s ...", parDic[COMPRESSION]) compCmd = "%s %s" % (parDic[COMPRESSION], stFn) compress_start = time.time() logger.debug("Compressing file with command: %s", compCmd) exitCode, stdOut = ngamsPlugInApi.execCmd(compCmd) #if (exitCode != 0): # msg ="Problems during archiving! Compressing the file failed. " +\ # "Error: %s" % str(stdOut).replace("/n", " ") # raise Exception, msg # If the compression fails, assume that it is because the file is not # compressible (although it could also be due to lack of disk space). if (exitCode == 0): if (parDic[COMPRESSION_EXT]): stFn = stFn + "." + parDic[COMPRESSION_EXT] comprExt = parDic[COMPRESSION_EXT] # Remember to update Staging Filename in the Request Properties # Object. reqPropsObj.setStagingFilename(stFn) # Handle mime-type if (parDic[TARG_MIME_TYPE]): format = parDic[TARG_MIME_TYPE] else: format = ngamsPlugInApi.determineMimeType( srvObj.getCfg(), stFn) compression = parDic[COMPRESSION] logger.debug("File compressed. Time: %.3fs", time.time() - compress_start) else: # Carry on with the original file. We take the original mime-type # as the target mime-type. format = reqPropsObj.getMimeType() compression = NO_COMPRESSION else: # Handle mime-type if (parDic[TARG_MIME_TYPE]): format = parDic[TARG_MIME_TYPE] else: format = reqPropsObj.getMimeType() compression = NO_COMPRESSION archFileSize = ngamsPlugInApi.getFileSize(reqPropsObj.getStagingFilename()) return uncomprSize, archFileSize, format, compression, comprExt
def ngamsBrotherPT9200DxPlugIn(srvObj, label, reqPropsObj=None): """ Driver for printing labels on the label printer Brother PT-9200DX. srvObj: Reference to instance of the NG/AMS Server class (ngamsServer). label: Label text to print (string). reqPropsObj: NG/AMS request properties object (ngamsReqProps). Returns: Void. """ plugInPars = srvObj.getCfg().getLabelPrinterPlugInPars() logger.info("Executing plug-in ngamsBrotherPT9200DxPlugIn with parameters: " + \ "%s - Label: %s ...", plugInPars, label) parDic = ngamsPlugInApi.parseRawPlugInPars(plugInPars) # Get the font bit pattern dictionary. fontDic = genFontsDictionary(parDic["font_file"]) # Generate the printer control code. printerCode = fontDic["Header"] for i in range(len(label)): if (not fontDic.has_key(label[i])): errMsg = "No font defintion for character: \"" + label[i] +\ "\" - in font definition file: " + parDic["font_file"] +\ " - cannot generate disk label: " + label ngamsPlugInApi.notify(srvObj, NGAMS_NOTIF_ERROR, "ngamsBrotherPT9200DxPlugIn: " +\ "ILLEGAL CHARACTER REQ. FOR PRINTING", errMsg) raise Exception(errMsg) printerCode = printerCode + fontDic[label[i]] printerCode = printerCode + fontDic["Trailer"] # Generate printer file, write the printer control code. tmpDir = ngamsPlugInApi.getTmpDir(srvObj.getCfg()) ngasId = srvObj.getHostId() printerFilename = os.path.normpath(tmpDir + "/ngamsLabel_" + ngasId + ".prn") fo = open(printerFilename, "w") fo.write(printerCode) fo.close() # Write the printer code file to the device. stat, out = ngamsPlugInApi.execCmd("cat " + printerFilename + " > " +\ parDic["dev"]) # This was previously excluded during the "test mode"; a proper way is by # having a configurable parameter if not 'keep_printer_file' in parDic: os.system("rm -f " + printerFilename) if (stat != 0): errMsg = "Problem occurred printing label! Error: " + str(out) ngamsPlugInApi.notify(srvObj, NGAMS_NOTIF_ERROR, "ngamsBrotherPT9200DxPlugIn: " +\ "PROBLEM PRINTING LABEL", errMsg) raise Exception(errMsg)
def ngamsFitsRegPlugIn(srvObj, reqPropsObj, parDic): """ Data Registration Plug-In to handle registration of FITS files. srvObj: Reference to NG/AMS Server Object (ngamsServer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). Returns: Standard NG/AMS Data Archiving Plug-In Status as generated by: ngamsPlugInApi.genDapiSuccessStat() (ngamsDapiStatus). """ logger.info("Plug-In registering file with URI: %s", reqPropsObj.getFileUri()) diskInfo = reqPropsObj.getTargDiskInfo() stageFile = reqPropsObj.getStagingFilename() # If the file is already compressed, we have to decompress it. procDir = "" if ((stageFile.find(".Z") != -1) or (stageFile.find(".gz") != -1)): workingFile, procDir = ngamsPlugInApi.prepProcFile(srvObj.getCfg(), stageFile) ngamsPlugInApi.execCmd("gunzip " + workingFile) if (workingFile.find(".Z") != -1): workingFile = workingFile[:-2] else: workingFile = workingFile[:-3] else: workingFile = stageFile # Check file (size + checksum). ngamsFitsPlugIn.checkFitsFileSize(workingFile) #ngamsFitsPlugIn.c_heckChecksum(parDic, workingFile) if 'skip_checksum' not in parDic: ngamsFitsPlugIn.checkFitsChecksum(reqPropsObj, workingFile) # Get various information about the file being handled. arcFile, dpId, dateDirName = ngamsFitsPlugIn.getDpIdInfo(workingFile) fileVersion, relPath, relFilename,\ complFilename, fileExists =\ ngamsPlugInApi.genFileInfoReg(srvObj.getDb(), srvObj.getCfg(), reqPropsObj, diskInfo, stageFile, dpId) # Generate status. logger.debug("Generating status ...") fileSize = ngamsPlugInApi.getFileSize(stageFile) if (stageFile.find(".Z") != -1): format = "application/x-cfits" compresion = "compress" elif (stageFile.find(".gz") != -1): format = "application/x-gfits" compresion = "gzip" else: format = "image/x-fits" compresion = "" uncomprSize = ngamsPlugInApi.getFileSize(workingFile) # Delete the processing directory (would be done later by the # Janitor Thread, but it is better to clean up explicitly). if (procDir): rmFile(procDir) logger.debug("Register Plug-In finished processing of file") return ngamsPlugInApi.genRegPiSuccessStat(diskInfo.getDiskId(),relFilename, dpId, fileVersion, format, fileSize, uncomprSize,compresion, relPath, diskInfo.getSlotId(), fileExists, complFilename)
def ngamsNgLogPlugIn(srvObj, reqPropsObj): """ Data Archiving Plug-In to handle archiving of NG/AMS (OLAS style) log files. srvObj: Reference to NG/AMS Server Object (ngamsServer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). Returns: Standard NG/AMS Data Archiving Plug-In Status as generated by: ngamsPlugInApi.genDapiSuccessStat() (ngamsDapiStatus). """ # For now the exception handling is pretty basic: # If something goes wrong during the handling it is tried to # move the temporary file to the Bad Files Area of the disk. logger.debug("Plug-In handling data for file: %s", os.path.basename(reqPropsObj.getFileUri())) diskInfo = reqPropsObj.getTargDiskInfo() stagingFilename = reqPropsObj.getStagingFilename() ext = os.path.splitext(stagingFilename)[1][1:] # Now, build final filename. We do that by taking the date of # the first log entry. # # 2001-07-09T14:37:59.563 [INFO] Logging properties defined ... # # Alternatively the first entry could be of the form, e.g.: # # 2003-12-29T09:21:57.608 [INFO] LOG-ROTATE: 1072689717 - \ # SYSTEM-ID: ngamsArchiveClient@ngasdev2 # # In the former case the Log ID is equal to the NGAS ID. In the latter # case, the Log ID is equal to the System ID. # # The final filename is built as follows: <Log ID>.<date>.<ext> # # The file_id is: <Log ID>.<date> fo = open(stagingFilename, "r") firstLine = fo.readline() fo.close() try: # Compress the log file. uncomprSize = ngamsPlugInApi.getFileSize(stagingFilename) compression = "gzip" logger.debug("Compressing file using: %s ...", compression) exitCode, stdOut = ngamsPlugInApi.execCmd("%s %s" %\ (compression, stagingFilename)) if (exitCode != 0): errMsg = "ngamsNgLogPlugIn: Problems during archiving! " +\ "Compressing the file failed" raise Exception(errMsg) stagingFilename = stagingFilename + ".gz" # Remember to update the Temporary Filename in the Request # Properties Object. reqPropsObj.setStagingFilename(stagingFilename) logger.debug("Log file compressed") # Parse first line of the log file. timeStamp = firstLine.split(" ")[0] date = timeStamp.split("T")[0] sysIdIdx = firstLine.find("SYSTEM-ID") if (sysIdIdx != -1): logId = firstLine[sysIdIdx + len("SYSTEM-ID:"):].strip().\ split(" ")[0] else: logId = srvObj.getHostId() fileId = logId + "." + timeStamp fileVersion, relPath, relFilename,\ complFilename, fileExists =\ ngamsPlugInApi.genFileInfo(srvObj.getDb(), srvObj.getCfg(), reqPropsObj, diskInfo, stagingFilename, fileId, fileId, [date]) # Generate status. logger.debug("Generating status ...") fformat = ngamsPlugInApi.determineMimeType(srvObj.getCfg(), stagingFilename) logger.debug("Determining file size ...") fileSize = ngamsPlugInApi.getFileSize(stagingFilename) return ngamsPlugInApi.genDapiSuccessStat( diskInfo.getDiskId(), relFilename, fileId, fileVersion, fformat, fileSize, uncomprSize, compression, relPath, diskInfo.getSlotId(), fileExists, complFilename) except Exception: raise Exception("ngamsNgLogPlugIn: Error handling log file: " +\ stagingFilename + ". Rejecting.")
def execCmd(cmd, failonerror=True): re = ngamsPlugInApi.execCmd(cmd) if (failonerror and (not os.WIFEXITED(re[0]))): errMsg = 'Fail to execute command: "%s". Exception: %s' % (cmd, re[1]) raise Exception(errMsg) return re