def ngamsExampleCacheCtrlPI(srvObj, cacheEntryObj): """ Example Cache Control Plug-in to control the cache holding based on an expiration time for each cached file. srvObj: Reference to NG/AMS Server Object (ngamsServer). cacheEntryObj: Cache Entry Object containing the information for the cached file (ngamsCacheEntry). Returns: Returns True if the file can be deleted from the cache, otherwise False (boolean). """ T = TRACE() try: global ngamsExampleCacheCtrlPI_maxCacheTime if (not ngamsExampleCacheCtrlPI_maxCacheTime): plugInPars = srvObj.getCfg().\ getVal("Caching[1].CacheControlPlugInPars") plugInParDic = ngamsPlugInApi.parseRawPlugInPars(plugInPars) try: _maxCacheTime = float(plugInParDic["max_cache_time"]) except: msg = "Missing plug-in parameter: max_cache_time" raise Exception, msg # Check if the cache time is exceeded. if ((time.time() - cacheEntryObj.getCacheTime()) > _maxCacheTime): return True return False except Exception: raise
def ngamsGLEAM_Phase_FilterPI(srvObj, plugInPars, filename, fileId, fileVersion = -1, reqPropsObj = None): """ srvObj: Reference to NG/AMS Server Object (ngamsServer). plugInPars: Parameters to take into account for the plug-in execution (string). fileId: File ID for file to test (string). filename: Filename of (complete) (string). fileVersion: Version of file to test (integer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). Returns: 0 if the file does not match, 1 if it matches the conditions (integer/0|1). """ if (not isGLEAMImage(fileId)): return 0 if (not srvObj.getDb().isLastVersion(fileId, fileVersion)): return 0 pars = "" if ((plugInPars != "") and (plugInPars != None)): pars = plugInPars elif (reqPropsObj != None): if (reqPropsObj.hasHttpPar("plug_in_pars")): pars = reqPropsObj.getHttpPar("plug_in_pars") parDic = ngamsPlugInApi.parseRawPlugInPars(pars) if (parDic.has_key('phase')): phase = int(parDic['phase']) else: return 1 # no need to check phase img_phase = None try: img_phase = getGLEAMPhase(filename) except Exception as exp: logger.warning("cannot get phase info from %s, file not added: %s" % (filename, str(exp))) return 0 if (phase == img_phase): return 1 else: return 0
def ngamsMimeTypeFilterPI(srvObj, plugInPars, filename, fileId, fileVersion=-1, reqPropsObj=None): """ Example Filter Plug-In used to filter on a given mime-type. In case the file referenced has the mime-type as specified in the plug-in parameters, the file being tested is selected. srvObj: Reference to NG/AMS Server Object (ngamsServer). plugInPars: Parameters to take into account for the plug-in execution (string). fileId: File ID for file to test (string). filename: Filename of (complete) (string). fileVersion: Version of file to test (integer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). Returns: 0 if the file does not match, 1 if it matches the conditions (integer/0|1). """ match = 0 # Parse plug-in parameters. parDic = [] pars = "" if ((plugInPars != "") and (plugInPars != None)): pars = plugInPars elif (reqPropsObj != None): if (reqPropsObj.hasHttpPar("plug_in_pars")): pars = reqPropsObj.getHttpPar("plug_in_pars") parDic = ngamsPlugInApi.parseRawPlugInPars(pars) if (not parDic.has_key("mime_types")): errMsg = "ngamsMimeTypeFilterPI: Missing Plug-In Parameter: " +\ "mime_types" raise Exception, errMsg # Perform the matching. refMimeTypes = parDic["mime_types"].split("|") actMimeType = ngamsPlugInApi.determineMimeType(srvObj.getCfg(), filename) for mt in refMimeTypes: if (actMimeType == mt.strip()): match = 1 return match
def ngamsAlmaCacheCtrlPI(srvObj, cacheEntryObj): """ srvObj: Reference to NG/AMS Server Object (ngamsServer). cacheEntryObj: Cache Entry Object containing the information for the cached file (ngamsCacheEntry). Returns: Returns True if the file can be deleted from the cache, otherwise False (boolean). """ plugInPars = srvObj.getCfg().getVal("Caching[1].CacheControlPlugInPars") plugInParDic = ngamsPlugInApi.parseRawPlugInPars(plugInPars) dblinks = plugInParDic["db_links_list"] dblinks_list = dblinks.split(":") for dblink in dblinks_list: query = "select count(*) from ngas_files@" + dblink srvObj.getDb().query2(query)
def ngamsExtractAlmaMultipart(srvObj, reqPropsObj, file_id): """ This DPPI extracts one part of a multipart/related message requested from the ALMA Archive. srvObj: Reference to instance of the NG/AMS Server class (ngamsServer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). file_id: Name of file to process (string). Returns: DPPI return status object (ngamsDppiStatus). Side effect: This DPPI works directly on the archived file, since it is read-only access. SPECIFIC DOCUMENTATION: This DPPI extracts one part of an ALMA multipart related message and returns a complete, self-consistent message containing a XML header and the requested cid. This version deals with VOTable headers and returns a new VOTable header containing only the RESOURCE element describing the requested (by cid) part. If no XML header can be found or if the XML header is not a VOTable header this plugin returns just the requested part, without any header. Example URL (single line): http://ngasdev1:7777/RETRIEVE ?file_id=X01/X7/X42& processing=ngamsAlmaMultipart& processing_pars='cid=<cid>' """ statusObj = ngamsDppiStatus.ngamsDppiStatus() cpart = 1 pars = 0 # initialize pars if (reqPropsObj.hasHttpPar("processing_pars")): pars = ngamsPlugInApi.parseRawPlugInPars(\ reqPropsObj.getHttpPar("processing_pars")) if pars and not pars.has_key('cid'): ext = '.cid' cpart = 1 # first part only by default elif pars and pars.has_key( 'cid'): # if processing_par 'cid' exists check its contents pass else: pass resFilename = file_id + ext try: mimeType = ngamsPlugInApi.determineMimeType(srvObj.getCfg(),\ resFilename) except: pass if ext == '.xml': mimeType = 'text/xml' else: mimeType = 'multipart/related' resObj = ngamsDppiStatus.ngamsDppiResult(NGAMS_PROC_DATA, mimeType, head, resFilename, '') statusObj.addResult(resObj) return statusObj
def _registerExec(srvObj, fileListDbmName, tmpFilePat, diskInfoDic, reqPropsObj=None): """ Register the files listed in the File List DBM (ngamsDbm), which match the mime-type(s) either specified in the 'mimeType' parameter, or if this is not specified, which match all the mime-types specified in the configuration file. When the registration procedure has been executed, the function sends an Email Notification message indicating which files were registered if the HTTP parameter 'notif_email' is given. The functions creates a File Info Objects per file handled and writes this in a temporary file, which is a DBM file. The keys in this DB is simply the file number in the sequence of files handled, pointing to a pickled ngamsFileInfo object. Each of the File Info Objects indicates if the file was registered or not. This is done by setting the tag of the File Info Object to one of the following values: REGISTERED: The file was successfully registered in the NGAS DB FAILED: The file was selected for registration but could not be properly registered because of inconsistencies. The status will be of the format: 'FAILED[: <reason>]'. REJECTED: A file found under the specified path directory was not accepted for cloning, usually because the mime-type was not correct. The status will be of the format: 'REJECTED[: <reason>]'. Note, that registration is comparable to archiving of files. For that reason a DAPI must be provided for each type of file that should be registered. If this is not fullfilled, the file registration will fail. Only files stored on one of the NGAS disks configured in the configuration file, are considered. Files stored in other locations are ignored. srvObj: Instance of NG/AMS Server object (ngamsServer). fileListDbmName: Name of a DBM containing the information about the files to be registered. Each element in the list is referred to by a key, which is a number. These points to a pickled list for each file containing the following information: [<Filename>, <Disk ID>, <Mime-Type>] The information for each disk concerned is also contained in the DB referred to by its Disk ID and by its mount point. The data is a pickled instance of the ngamsDiskInfo class (string). tmpFilePat: Pattern for temporary files used during the registration process (string). diskInfoDic: Dictionary with Disk IDs as keys pointing to the info about the disk (dictionary/ngamsDiskInfo). reqPropsObj: If an NG/AMS Request Properties Object is given, the Request Status will be updated as the request is carried out (ngamsReqProps). Returns: Void. """ emailNotif = 0 if (reqPropsObj): if (reqPropsObj.hasHttpPar("notif_email")): emailNotif = 1 # Create the temporary BSD DB to contain the information for the # Email Notification Message. if (emailNotif): regDbmName = tmpFilePat + "_NOTIF_EMAIL" regDbm = ngamsDbm.ngamsDbm(regDbmName, writePerm=1) # Open the DBM containing the list of files to (possibly) register. fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm=1) # Want to parse files in alphabetical order. # TODO: Portatibility issue. Try to avoid UNIX shell commands for sorting. tmpFileList = tmpFilePat + "_FILE_LIST" rmFile(tmpFileList) with open(tmpFileList, "wb") as fo: fileListDbm.initKeyPtr() while (1): dbmKey, fileInfo = fileListDbm.getNext() if (not dbmKey): break fo.write(dbmKey + b"\n") sortFileList = tmpFilePat + "_SORT_FILE_LIST" rmFile(sortFileList) shellCmd = "sort %s > %s" % (tmpFileList, sortFileList) stat, out, err = ngamsCore.execCmd(shellCmd) if (stat != 0): raise Exception("Error executing command: %s. Error: %s, %s" %\ (shellCmd, str(out), str(err))) rmFile(tmpFileList) # Go through each file in the list, check if the mime-type is among the # ones, which apply for registration. If yes try to register the file # by invoking the corresponding DAPI on the file. fileRegCount = 0 fileFailCount = 0 fileRejectCount = 0 regTimeAccu = 0.0 fileCount = 0 fo = open(sortFileList) run = 1 while (run): reg_start = time.time() dbmKey = fo.readline() if (dbmKey.strip() == ""): run = 0 continue fileInfo = fileListDbm.get(dbmKey[0:-1]) filename = fileInfo[0] diskId = fileInfo[1] mimeType = fileInfo[2] # Register the file. Check first, that exactly this file is # not already registered. In case it is, the file will be rejected. regPi = srvObj.getCfg().register_plugins[mimeType] logger.debug("Plugin found for %s: %s", mimeType, regPi) params = ngamsPlugInApi.parseRawPlugInPars(regPi.pars) tmpReqPropsObj = ngamsReqProps.ngamsReqProps().\ setMimeType(mimeType).\ setStagingFilename(filename).\ setTargDiskInfo(diskInfoDic[diskId]).\ setHttpMethod(NGAMS_HTTP_GET).\ setCmd(NGAMS_REGISTER_CMD).\ setSize(os.path.getsize(filename)).\ setFileUri(filename).\ setNoReplication(1) tmpFileObj = ngamsFileInfo.ngamsFileInfo() try: # Invoke Registration Plug-In. piName = regPi.name plugInMethod = loadPlugInEntryPoint(piName) piRes = plugInMethod(srvObj, tmpReqPropsObj, params) del tmpReqPropsObj # Check if this file is already registered on this disk. In case # yes, it is not registered again. files = srvObj.db.getFileSummary1(srvObj.getHostId(), [piRes.getDiskId()], [piRes.getFileId()]) fileRegistered = 0 for tmpFileInfo in files: tmpMtPt = tmpFileInfo[ngamsDbCore.SUM1_MT_PT] tmpFilename = tmpFileInfo[ngamsDbCore.SUM1_FILENAME] tmpComplFilename = os.path.normpath(tmpMtPt + "/" +\ tmpFilename) if (tmpComplFilename == filename): fileRegistered = 1 break if (fileRegistered): fileRejectCount += 1 tmpMsgForm = "REJECTED: File with File ID/Version: %s/%d " +\ "and path: %s is already registered on disk " +\ "with Disk ID: %s" tmpMsg = tmpMsgForm % (piRes.getFileId(), piRes.getFileVersion(), filename, piRes.getDiskId()) logger.warning(tmpMsg + ". File is not registered again.") if (emailNotif): tmpFileObj.\ setDiskId(diskId).setFilename(filename).\ setTag(tmpMsg) regDbm.addIncKey(tmpFileObj) if (reqPropsObj): reqPropsObj.incActualCount(1) ngamsHighLevelLib.stdReqTimeStatUpdate( srvObj, reqPropsObj, regTimeAccu) regTimeAccu += time.time() - reg_start fileCount += 1 continue # Calculate checksum. We maintain the old name for backwards # compatibility crc_variant = srvObj.cfg.getCRCVariant() if crc_variant == ngamsFileUtils.CHECKSUM_CRC32_INCONSISTENT: crc_variant = 'ngamsGenCrc32' checksum = ngamsFileUtils.get_checksum(65536, filename, crc_variant) or '' # Move file and update information about file in the NGAS DB. mvFile(filename, piRes.getCompleteFilename()) ngamsArchiveUtils.updateFileInfoDb(srvObj, piRes, checksum, crc_variant) ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), piRes) ngamsLib.makeFileReadOnly(piRes.getCompleteFilename()) if (emailNotif): uncomprSize = piRes.getUncomprSize() ingestDate = time.time() creDateSecs = getFileCreationTime(filename) tmpFileObj.\ setDiskId(diskId).\ setFilename(filename).\ setFileId(piRes.getFileId()).\ setFileVersion(piRes.getFileVersion()).\ setFormat(piRes.getFormat()).\ setFileSize(piRes.getFileSize()).\ setUncompressedFileSize(uncomprSize).\ setCompression(piRes.getCompression()).\ setIngestionDate(ingestDate).\ setIgnore(0).\ setChecksum(checksum).\ setChecksumPlugIn(crc_variant).\ setFileStatus(NGAMS_FILE_STATUS_OK).\ setCreationDate(creDateSecs).\ setTag("REGISTERED") fileRegCount += 1 # If running as a cache archive, update the Cache New Files DBM # with the information about the new file. if (srvObj.getCachingActive()): fileVer = fio.getFileVersion() ngamsCacheControlThread.addEntryNewFilesDbm( srvObj, diskId, piRes.getFileId(), fileVer, filename) # Generate a confirmation log entry. msg = genLog("NGAMS_INFO_FILE_REGISTERED", [ filename, piRes.getFileId(), piRes.getFileVersion(), piRes.getFormat() ]) time.sleep(0.005) regTime = time.time() - reg_start msg = msg + ". Time: %.3fs." % (regTime) logger.info(msg, extra={'to_syslog': 1}) except Exception as e: errMsg = genLog("NGAMS_ER_FILE_REG_FAILED", [filename, str(e)]) logger.error(errMsg) if (emailNotif): tmpFileObj.\ setDiskId(diskId).setFilename(filename).\ setTag(errMsg) fileFailCount += 1 regTime = time.time() - reg_start # TODO (rtobar, 2016-01): Why don't we raise an exception here? # Otherwise the command appears as successful on the # client-side # Add the file information in the registration report. if (emailNotif): regDbm.addIncKey(tmpFileObj) # Update request status time information. regTimeAccu += regTime if (reqPropsObj): reqPropsObj.incActualCount(1) ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj, regTimeAccu) fileCount += 1 fo.close() rmFile(sortFileList) if (emailNotif): regDbm.sync() del fileListDbm rmFile(fileListDbmName + "*") # Final update of the Request Status. if (reqPropsObj): if (reqPropsObj.getExpectedCount() and reqPropsObj.getActualCount()): complPercent = (100.0 * (float(reqPropsObj.getActualCount()) / float(reqPropsObj.getExpectedCount()))) else: complPercent = 100.0 reqPropsObj.setCompletionPercent(complPercent, 1) reqPropsObj.setCompletionTime(1) srvObj.updateRequestDb(reqPropsObj) # Send Register Report with list of files cloned to a possible # requestor(select) of this. if (emailNotif): xmlStat = 0 if (xmlStat): # Create an instance of the File List Class, used to store the # Registration Report. regStat = ngamsFileList.\ ngamsFileList("FILE_REGISTRATION_STATUS", "Status report for file " +\ "registration") # Loop over the file objects in the BSD DB and add these # in the status object. regDbm.initKeyPtr() while (1): key, tmpFileObj = regDbm.getNext() if (not key): break regStat.addFileInfoObj(tmpFileObj) # Set overall status of registration procedure. regStat.setStatus("Files Found: " + str(fileCount + 1) + ", "+\ "Files Registered: " + str(fileRegCount) +\ ", " +\ "Files Failed: " + str(fileFailCount) + ", " +\ "Files Rejected: " + str(fileRejectCount)) status = srvObj.genStatus(NGAMS_SUCCESS, "REGISTER command status report").\ addFileList(regStat) statRep = status.genXmlDoc() statRep = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, statRep) mimeType = NGAMS_XML_MT else: # Generate a 'simple' ASCII report. statRep = tmpFilePat + "_NOTIF_EMAIL.txt" fo = open(statRep, "w") if (reqPropsObj): path = reqPropsObj.getHttpPar("path") else: path = "-----" if (fileCount): timePerFile = (regTimeAccu / fileCount) else: timePerFile = 0 tmpFormat = "REGISTER STATUS REPORT:\n\n" +\ "==Summary:\n\n" +\ "Date: %s\n" +\ "NGAS Host: %s\n" +\ "Search Path: %s\n" +\ "Total Number of Files: %d\n" +\ "Number of Registered Files: %d\n" +\ "Number of Failed Files: %d\n" +\ "Number of Rejected Files: %d\n" +\ "Total processing time (s): %.3f\n" +\ "Handling time per file (s): %.3f\n\n" +\ "==File List:\n\n" fo.write(tmpFormat % (toiso8601(), srvObj.getHostId(), path, fileCount, fileRegCount, fileFailCount, fileRejectCount, regTimeAccu, timePerFile)) tmpFormat = "%-80s %-32s %-3s %-10s\n" fo.write(tmpFormat %\ ("Filename", "File ID", "Ver", "Status")) fo.write(tmpFormat % (80 * "-", 32 * "-", 3 * "-", 10 * "-")) regDbm.initKeyPtr() while (1): key, tmpFileObj = regDbm.getNext() if (not key): break mtPt = diskInfoDic[tmpFileObj.getDiskId()].getMountPoint() filename = os.path.normpath(mtPt + "/" +\ tmpFileObj.getFilename()) line = tmpFormat %\ (filename, tmpFileObj.getFileId(), str(tmpFileObj.getFileVersion()), tmpFileObj.getTag()) fo.write(line) fo.write(128 * "-") fo.write("\n\n==END\n") fo.close() mimeType = NGAMS_TEXT_MT # Send out the status report. emailAdrList = reqPropsObj.getHttpPar("notif_email").split(",") attachmentName = "RegisterStatusReport" if (reqPropsObj.hasHttpPar("path")): attachmentName += "-" + reqPropsObj.getHttpPar("path").\ replace("/", "_") ngamsNotification.notify(srvObj.host_id, srvObj.cfg, NGAMS_NOTIF_INFO, "REGISTER STATUS REPORT", statRep, recList=emailAdrList, force=1, contentType=mimeType, attachmentName=attachmentName) del regDbm rmFile(regDbmName + "*") rmFile(statRep) # Generate final status log + exit. if (fileCount > 0): timePerFile = (regTimeAccu / fileCount) else: timePerFile = 0.0 msg = "Registration procedure finished processing Register Request - " + \ "terminating. Files handled: %d. Total time: %.3fs. " + \ "Average time per file: %.3fs." logger.debug(msg, fileCount, regTimeAccu, timePerFile)
def ngamsGLEAM_RRI_FilterPI(srvObj, plugInPars, filename, fileId, fileVersion=-1, reqPropsObj=None): """ srvObj: Reference to NG/AMS Server Object (ngamsServer). plugInPars: Parameters to take into account for the plug-in execution (string). fileId: File ID for file to test (string). filename: Filename of (complete) (string). fileVersion: Version of file to test (integer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). Returns: 0 if the file does not match, 1 if it matches the conditions (integer/0|1). """ match = 0 fn, fext = os.path.splitext(fileId) if (fext.lower() in file_ext and # only send FITS files, no measurement sets _shouldSend(fileId) and # # only send files satisfying certain string pattern criteria srvObj.getDb().isLastVersion( fileId, fileVersion)): # only send the (known) latest version parDic = [] pars = "" if ((plugInPars != "") and (plugInPars != None)): pars = plugInPars elif (reqPropsObj != None): if (reqPropsObj.hasHttpPar("plug_in_pars")): pars = reqPropsObj.getHttpPar("plug_in_pars") parDic = ngamsPlugInApi.parseRawPlugInPars(pars) if (not parDic.has_key("remote_host") or not parDic.has_key("remote_port")): errMsg = "ngamsGLEAM_VUW_FilterPI: Missing Plug-In Parameter: " +\ "remote_host / remote_port" #raise Exception, errMsg logger.error(errMsg) return 1 # matched as if the remote checking is done host = parDic["remote_host"] sport = parDic["remote_port"] if (not sport.isdigit()): errMsg = "ngamsGLEAM_VUW_FilterPI: Invalid port number: " + sport logger.error(errMsg) return 1 # matched as if the filter does not exist port = int(sport) # Perform the matching. client = ngamsPClient.ngamsPClient(host, port, timeOut=NGAMS_SOCK_TIMEOUT_DEF) try: if (fileVersion == -1): fileVersion = 1 rest = client.get_status(NGAMS_STATUS_CMD, pars=[["file_id", fileId], ["file_version", fileVersion]]) # since the queue will be sorted based on ingestion date, this will ensure the versions are sent by order: # e.g. version1, version2, version3, otherwise, this method will have disordered versions sent if (rest.getStatus().find(NGAMS_FAILURE) != -1): return 1 # matched since file id does not exist except Exception as e: errMsg = "Error occurred during checking remote file status " +\ "ngamsGLEAM_VUW_FilterPI. Exception: " + str(e) logger.error(errMsg) return 1 # matched as if the filter does not exist #info(5, "filter return status = " + rest.getStatus()) #info(4, "filter match = " + str(match)) return match
def _registerExec(srvObj, fileListDbmName, tmpFilePat, diskInfoDic, reqPropsObj = None): """ Register the files listed in the File List DBM (ngamsDbm), which match the mime-type(s) either specified in the 'mimeType' parameter, or if this is not specified, which match all the mime-types specified in the configuration file. When the registration procedure has been executed, the function sends an Email Notification message indicating which files were registered if the HTTP parameter 'notif_email' is given. The functions creates a File Info Objects per file handled and writes this in a temporary file, which is a DBM file. The keys in this DB is simply the file number in the sequence of files handled, pointing to a pickled ngamsFileInfo object. Each of the File Info Objects indicates if the file was registered or not. This is done by setting the tag of the File Info Object to one of the following values: REGISTERED: The file was successfully registered in the NGAS DB FAILED: The file was selected for registration but could not be properly registered because of inconsistencies. The status will be of the format: 'FAILED[: <reason>]'. REJECTED: A file found under the specified path directory was not accepted for cloning, usually because the mime-type was not correct. The status will be of the format: 'REJECTED[: <reason>]'. Note, that registration is comparable to archiving of files. For that reason a DAPI must be provided for each type of file that should be registered. If this is not fullfilled, the file registration will fail. Only files stored on one of the NGAS disks configured in the configuration file, are considered. Files stored in other locations are ignored. srvObj: Instance of NG/AMS Server object (ngamsServer). fileListDbmName: Name of a DBM containing the information about the files to be registered. Each element in the list is referred to by a key, which is a number. These points to a pickled list for each file containing the following information: [<Filename>, <Disk ID>, <Mime-Type>] The information for each disk concerned is also contained in the DB referred to by its Disk ID and by its mount point. The data is a pickled instance of the ngamsDiskInfo class (string). tmpFilePat: Pattern for temporary files used during the registration process (string). diskInfoDic: Dictionary with Disk IDs as keys pointing to the info about the disk (dictionary/ngamsDiskInfo). reqPropsObj: If an NG/AMS Request Properties Object is given, the Request Status will be updated as the request is carried out (ngamsReqProps). Returns: Void. """ T = TRACE() emailNotif = 0 if (reqPropsObj): if (reqPropsObj.hasHttpPar("notif_email")): emailNotif = 1 # Create the temporary BSD DB to contain the information for the # Email Notification Message. if (emailNotif): regDbmName = tmpFilePat + "_NOTIF_EMAIL" regDbm = ngamsDbm.ngamsDbm(regDbmName, writePerm = 1) # Open the DBM containing the list of files to (possibly) register. fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm = 1) # Want to parse files in alphabetical order. # TODO: Portatibility issue. Try to avoid UNIX shell commands for sorting. tmpFileList = tmpFilePat + "_FILE_LIST" rmFile(tmpFileList) fo = open(tmpFileList, "w") fileListDbm.initKeyPtr() while (1): dbmKey, fileInfo = fileListDbm.getNext() if (not dbmKey): break fo.write(dbmKey + "\n") fo.close() sortFileList = tmpFilePat + "_SORT_FILE_LIST" rmFile(sortFileList) shellCmd = "sort %s > %s" % (tmpFileList, sortFileList) stat, out = commands.getstatusoutput(shellCmd) if (stat != 0): raise Exception, "Error executing command: %s. Error: %s" %\ (shellCmd, str(out)) rmFile(tmpFileList) # Go through each file in the list, check if the mime-type is among the # ones, which apply for registration. If yes try to register the file # by invoking the corresponding DAPI on the file. checksumPlugIn = srvObj.getCfg().getChecksumPlugIn().strip() fileRegCount = 0 fileFailCount = 0 fileRejectCount = 0 regTimeAccu = 0.0 fileCount = 0 fo = open(sortFileList) run = 1 while (run): reg_start = time.time() dbmKey = fo.readline() if (dbmKey.strip() == ""): run = 0 continue fileInfo = fileListDbm.get(dbmKey[0:-1]) filename = fileInfo[0] diskId = fileInfo[1] mimeType = fileInfo[2] # Register the file. Check first, that exactly this file is # not already registered. In case it is, the file will be rejected. regPi = srvObj.getCfg().register_plugins[mimeType] logger.debug("Plugin found for %s: %s", mimeType, regPi) params = ngamsPlugInApi.parseRawPlugInPars(regPi.pars) tmpReqPropsObj = ngamsReqProps.ngamsReqProps().\ setMimeType(mimeType).\ setStagingFilename(filename).\ setTargDiskInfo(diskInfoDic[diskId]).\ setHttpMethod(NGAMS_HTTP_GET).\ setCmd(NGAMS_REGISTER_CMD).\ setSize(os.path.getsize(filename)).\ setFileUri(filename).\ setNoReplication(1) tmpFileObj = ngamsFileInfo.ngamsFileInfo() try: # Invoke Registration Plug-In. piName = regPi.name plugInMethod = loadPlugInEntryPoint(piName) piRes = plugInMethod(srvObj, tmpReqPropsObj, params) del tmpReqPropsObj # Check if this file is already registered on this disk. In case # yes, it is not registered again. files = srvObj.db.getFileSummary1(srvObj.getHostId(), [piRes.getDiskId()], [piRes.getFileId()]) fileRegistered = 0 for tmpFileInfo in files: tmpMtPt = tmpFileInfo[ngamsDbCore.SUM1_MT_PT] tmpFilename = tmpFileInfo[ngamsDbCore.SUM1_FILENAME] tmpComplFilename = os.path.normpath(tmpMtPt + "/" +\ tmpFilename) if (tmpComplFilename == filename): fileRegistered = 1 break if (fileRegistered): fileRejectCount += 1 tmpMsgForm = "REJECTED: File with File ID/Version: %s/%d " +\ "and path: %s is already registered on disk " +\ "with Disk ID: %s" tmpMsg = tmpMsgForm % (piRes.getFileId(), piRes.getFileVersion(), filename, piRes.getDiskId()) logger.warning(tmpMsg + ". File is not registered again.") if (emailNotif): tmpFileObj.\ setDiskId(diskId).setFilename(filename).\ setTag(tmpMsg) regDbm.addIncKey(tmpFileObj) if (reqPropsObj): reqPropsObj.incActualCount(1) ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj, regTimeAccu) regTimeAccu += time.time() - reg_start fileCount += 1 continue # Calculate checksum (if plug-in specified). if (checksumPlugIn != ""): logger.debug("Invoking Checksum Plug-In: %s to handle file: %s", checksumPlugIn, filename) plugInMethod = loadPlugInEntryPoint(checksumPlugIn) checksum = plugInMethod(srvObj, filename, 0) else: checksum = "" # Move file and update information about file in the NGAS DB. mvFile(filename, piRes.getCompleteFilename()) ngamsArchiveUtils.updateFileInfoDb(srvObj, piRes, checksum, checksumPlugIn) ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), piRes) ngamsLib.makeFileReadOnly(piRes.getCompleteFilename()) if (emailNotif): uncomprSize = piRes.getUncomprSize() ingestDate = time.time() creDateSecs = getFileCreationTime(filename) tmpFileObj.\ setDiskId(diskId).\ setFilename(filename).\ setFileId(piRes.getFileId()).\ setFileVersion(piRes.getFileVersion()).\ setFormat(piRes.getFormat()).\ setFileSize(piRes.getFileSize()).\ setUncompressedFileSize(uncomprSize).\ setCompression(piRes.getCompression()).\ setIngestionDate(ingestDate).\ setIgnore(0).\ setChecksum(checksum).\ setChecksumPlugIn(checksumPlugIn).\ setFileStatus(NGAMS_FILE_STATUS_OK).\ setCreationDate(creDateSecs).\ setTag("REGISTERED") fileRegCount += 1 # If running as a cache archive, update the Cache New Files DBM # with the information about the new file. if (srvObj.getCachingActive()): fileVer = fio.getFileVersion() ngamsCacheControlThread.addEntryNewFilesDbm(srvObj, diskId, piRes.getFileId(), fileVer, filename) # Generate a confirmation log entry. msg = genLog("NGAMS_INFO_FILE_REGISTERED", [filename, piRes.getFileId(), piRes.getFileVersion(), piRes.getFormat()]) time.sleep(0.005) regTime = time.time() - reg_start msg = msg + ". Time: %.3fs." % (regTime) logger.info(msg, extra={'to_syslog': 1}) except Exception, e: errMsg = genLog("NGAMS_ER_FILE_REG_FAILED", [filename, str(e)]) logger.error(errMsg) if (emailNotif): tmpFileObj.\ setDiskId(diskId).setFilename(filename).\ setTag(errMsg) fileFailCount += 1 regTime = time.time() - reg_start # TODO (rtobar, 2016-01): Why don't we raise an exception here? # Otherwise the command appears as successful on the # client-side # Add the file information in the registration report. if (emailNotif): regDbm.addIncKey(tmpFileObj) # Update request status time information. regTimeAccu += regTime if (reqPropsObj): reqPropsObj.incActualCount(1) ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj, regTimeAccu) fileCount += 1
def ngamsMWA_Compress_JobPlugin(srvObj, plugInPars, filename, fileId, fileVersion, diskId): """ srvObj: Reference to NG/AMS Server Object (ngamsServer). plugInPars: Parameters to take into account for the plug-in execution (string).(e.g. scale_factor=4,threshold=1E-5) fileId: File ID for file to test (string). filename: Filename of (complete) (string). fileVersion: Version of file to test (integer). Returns: the return code of the compression plugin (integer). """ pars = "" sf = 4 # scaling factor th = 1E-5 # threshold bins = 0 remove_uc = 0 if ((plugInPars != "") and (plugInPars != None)): pars = plugInPars parDic = ngamsPlugInApi.parseRawPlugInPars(pars) if (parDic.has_key('scale_factor')): sf = int(parDic['scale_factor']) if (parDic.has_key('threshold')): th = float(parDic['threshold']) if (parDic.has_key('bins')): bins = int(parDic['bins']) if (parDic.has_key('remove_uc')): remove_uc = int(parDic['remove_uc']) if (bins): binstr = '-h %d' % bins else: binstr = '' newfn = '%s/%s' % (work_dir, os.path.basename(filename)) archive_host = get_contact_ip( srvObj.getCfg() ) # archive host must be on the same machine as the data mover or job runner cmd = "%s -d %d %s %s %s" % (uvcompress, sf, binstr, filename, newfn) cmd1 = "%s -host %s -port 7777 -fileUri %s -cmd QARCHIVE -mimeType application/octet-stream " % ( archive_client, archive_host, newfn) cmd2 = "curl http://%s:7777/DISCARD?file_id=%s\\&file_version=%d\\&disk_id=%s\\&execute=1" % ( archive_host, fileId, fileVersion, diskId) cmd3 = "rm %s" % newfn if (debug): logger.debug('*******************************************') logger.debug(cmd) logger.debug(cmd1) if (remove_uc): logger.debug(cmd2) logger.debug(cmd3) logger.debug('*******************************************') return (0, 'Compressed OK') else: re = commands.getstatusoutput(cmd) if (0 == re[0]): if (not bins): retstr = re[1].split('\n')[-1] # just get the elapsed time else: retstr = re[1].split('------ Histogram ------\n')[1] # archive it back # TODO - enable time out!! re = commands.getstatusoutput(cmd1) if (0 == re[0]): logger.debug('Successfully re-archived the compressed file %s', newfn) else: logger.error('Fail to re-archive compressed file %s: %s', newfn, re[1]) return (re[0], re[1]) if (remove_uc): # remove the uncompressed file if necessary re = commands.getstatusoutput(cmd2) if (0 == re[0]): logger.debug( 'Successfully DISCARDED the uncompressed file %s', filename) else: logger.warning('Fail to DISCARD the uncompressed file %s', filename) # remove the temp file re = commands.getstatusoutput(cmd3) if (0 != re[0]): logger.warning('Fail to remove the temp compressed file %s', newfn) return (0, retstr) else: logger.error('Fail to compress file %s: %s', filename, re[1]) return (re[0], re[1])
def ngamsMWA_MIT_NoTape_FilterPI(srvObj, plugInPars, filename, fileId, fileVersion=-1, reqPropsObj=None): """ srvObj: Reference to NG/AMS Server Object (ngamsServer). plugInPars: Parameters to take into account for the plug-in execution (string). fileId: File ID for file to test (string). filename: Filename of (complete) (string). fileVersion: Version of file to test (integer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). Returns: 0 if the file does not match, 1 if it matches the conditions (integer/0|1). """ match = 0 projectId = '' try: projectId = pyfits.getval(filename, 'PROJID') except: err = "Did not find keyword PROJID in FITS file or PROJID illegal" errMsg = genLog( "NGAMS_ER_DAPI_BAD_FILE", [os.path.basename(filename), "ngamsMWA_MIT_FilterPlugIn", err]) #raise Exception, errMsg #so still possible to deliver if the file is not there yet """ if (projectId == "'C105'" or projectId == "'C106'"): return 0 """ # Parse plug-in parameters. pars = "" if ((plugInPars != "") and (plugInPars != None)): pars = plugInPars elif (reqPropsObj != None): if (reqPropsObj.hasHttpPar("plug_in_pars")): pars = reqPropsObj.getHttpPar("plug_in_pars") parDic = ngamsPlugInApi.parseRawPlugInPars(pars) if (not parDic.has_key("remote_host") or not parDic.has_key("remote_port") or not parDic.has_key("project_id")): errMsg = "ngamsMWACheckRemoteFilterPlugin: Missing Plug-In Parameter: " +\ "remote_host / remote_port / project_id" #raise Exception, errMsg logger.error(errMsg) return 1 # matched as if the filter did not exist host = parDic["remote_host"] sport = parDic["remote_port"] proj_ids = parDic["project_id"] if (proj_ids and len(proj_ids)): for proj_id in proj_ids.split(proj_separator): eor_list.append("'%s'" % proj_id) if (not (projectId in eor_list)): logger.debug('File %s is not EOR project', fileId) return 0 if (not sport.isdigit()): errMsg = "ngamsMWACheckRemoteFilterPlugin: Invalid port number: " + sport logger.error(errMsg) return 1 # matched as if the filter does not exist port = int(sport) # Perform the matching. client = ngamsPClient.ngamsPClient(host, port, timeOut=NGAMS_SOCK_TIMEOUT_DEF) try: rest = client.get_status(NGAMS_STATUS_CMD, pars=[["file_id", fileId]]) except Exception: errMsg = "Error occurred during checking remote file status " +\ "ngamsMWACheckRemoteFilterPlugin" logger.exception(errMsg) return 1 # matched as if the filter does not exist #info(5, "filter return status = " + rest.getStatus()) if (rest.getStatus().find(NGAMS_FAILURE) != -1): tname = threading.current_thread().name beingSent = srvObj._subscrDeliveryFileDic.values() for fi in beingSent: if (srvObj._subscrDeliveryFileDic[tname] != fi and fi[0] == fileId and fi[2] == fileVersion): return 0 # this file is currently being sent, so do not send it again match = 1 #info(4, "filter match = " + str(match)) return match
def ngamsMWACheckRemoteFilterPlugin(srvObj, plugInPars, filename, fileId, fileVersion=-1, reqPropsObj=None): """ Example Filter Plug-In used to filter on a given mime-type. In case the file referenced has the mime-type as specified in the plug-in parameters, the file being tested is selected. srvObj: Reference to NG/AMS Server Object (ngamsServer). plugInPars: Parameters to take into account for the plug-in execution (string). fileId: File ID for file to test (string). filename: Filename of (complete) (string). fileVersion: Version of file to test (integer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). Returns: 0 if the file does not match, 1 if it matches the conditions (integer/0|1). """ match = 0 # Parse plug-in parameters. parDic = [] pars = "" if ((plugInPars != "") and (plugInPars != None)): pars = plugInPars elif (reqPropsObj != None): if (reqPropsObj.hasHttpPar("plug_in_pars")): pars = reqPropsObj.getHttpPar("plug_in_pars") parDic = ngamsPlugInApi.parseRawPlugInPars(pars) if (not parDic.has_key("remote_host") or not parDic.has_key("remote_port")): errMsg = "ngamsMWACheckRemoteFilterPlugin: Missing Plug-In Parameter: " +\ "remote_host and/or remote_port" #raise Exception, errMsg logger.error(errMsg) return 1 # matched as if the filter does not exist host = parDic["remote_host"] sport = parDic["remote_port"] if (not sport.isdigit()): errMsg = "ngamsMWACheckRemoteFilterPlugin: Invalid port number: " + sport logger.error(errMsg) return 1 # matched as if the filter does not exist port = int(sport) # Perform the matching. client = ngamsPClient.ngamsPClient(host, port) try: rest = client.get_status(NGAMS_STATUS_CMD, pars=[["file_id", fileId]]) except Exception: errMsg = "Error occurred during checking remote file status " +\ "ngamsMWACheckRemoteFilterPlugin" logger.exception(errMsg) return 1 # matched as if the filter does not exist logger.debug("filter return status = %s", rest.getStatus()) if (rest.getStatus().find(NGAMS_FAILURE) != -1): match = 1 logger.debug("filter match = %s", str(match)) return match
def ngamsMWA_MIT_FilterPlugin(srvObj, plugInPars, filename, fileId, fileVersion=-1, reqPropsObj=None): """ srvObj: Reference to NG/AMS Server Object (ngamsServer). plugInPars: Parameters to take into account for the plug-in execution (string). fileId: File ID for file to test (string). filename: Filename of (complete) (string). fileVersion: Version of file to test (integer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). Returns: 0 if the file does not match, 1 if it matches the conditions (integer/0|1). """ pars = "" if ((plugInPars != "") and (plugInPars != None)): pars = plugInPars elif (reqPropsObj != None): if (reqPropsObj.hasHttpPar("plug_in_pars")): pars = reqPropsObj.getHttpPar("plug_in_pars") parDic = ngamsPlugInApi.parseRawPlugInPars(pars) if (not parDic.has_key("remote_host") or not parDic.has_key("remote_port") or not parDic.has_key("project_id")): errMsg = "ngamsMWACheckRemoteFilterPlugin: Missing Plug-In Parameter: " +\ "remote_host / remote_port / project_id" logger.error(errMsg) return 0 proj_ids = parDic["project_id"] if (not (proj_ids and len(proj_ids))): return 0 for proj_id in proj_ids.split(proj_separator): eor_list.append("'%s'" % proj_id) fspi = srvObj.getCfg().getFileStagingPlugIn() if (not fspi): offline = -1 else: logger.info("Invoking FSPI.isFileOffline: %s to check file: %s", fspi, filename) isFileOffline = loadPlugInEntryPoint(fspi, 'isFileOffline') offline = isFileOffline(filename) try: if (offline == 1 or offline == -1): # if the file is on Tape or query error, query db instead, otherwise implicit tape staging will block all other threads!! logger.debug('File %s appears on Tape, connect to MWA DB to check', filename) projId = getProjectIdFromMWADB(fileId) if (not projId or projId == ''): logger.error('Cannot get project id from MWA DB for file %s', fileId) return 0 projectId = "'%s'" % projId # add single quote to be consistent with FITS header keywords else: projectId = pyfits.getval(filename, 'PROJID') except: err = "Did not find keyword PROJID in FITS file or PROJID illegal" errMsg = genLog( "NGAMS_ER_DAPI_BAD_FILE", [os.path.basename(filename), "ngamsMWA_MIT_FilterPlugIn", err]) return 0 if (projectId in eor_list): logger.debug('File %s added', fileId) return 1 else: return 0
def ngamsMWA_MIT_Priority_FilterPI(srvObj, plugInPars, filename, fileId, fileVersion=-1, reqPropsObj=None): """ srvObj: Reference to NG/AMS Server Object (ngamsServer). plugInPars: Parameters to take into account for the plug-in execution (string). fileId: File ID for file to test (string). filename: Filename of (complete) (string). fileVersion: Version of file to test (integer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). Returns: 0 if the file does not match, 1 if it matches the conditions (integer/0|1). """ match = 0 obsId = fileId.split('_')[0] if (not obsId in obs_list): return 0 # Parse plug-in parameters. pars = "" if ((plugInPars != "") and (plugInPars != None)): pars = plugInPars elif (reqPropsObj != None): if (reqPropsObj.hasHttpPar("plug_in_pars")): pars = reqPropsObj.getHttpPar("plug_in_pars") parDic = ngamsPlugInApi.parseRawPlugInPars(pars) if (not parDic.has_key("remote_host") or not parDic.has_key("remote_port")): errMsg = "ngamsMWACheckRemoteFilterPlugin: Missing Plug-In Parameter: " +\ "remote_host / remote_port" #raise Exception, errMsg logger.error(errMsg) return 1 # matched as if the filter did not exist host = parDic["remote_host"] sport = parDic["remote_port"] if (not sport.isdigit()): errMsg = "ngamsMWACheckRemoteFilterPlugin: Invalid port number: " + sport logger.error(errMsg) return 1 # matched as if the filter does not exist port = int(sport) # Perform the matching. client = ngamsPClient.ngamsPClient(host, port, timeOut=NGAMS_SOCK_TIMEOUT_DEF) try: rest = client.get_status(NGAMS_STATUS_CMD, pars=[["file_id", fileId]]) except Exception: errMsg = "Error occurred during checking remote file status " +\ "ngamsMWACheckRemoteFilterPlugin" logger.exception(errMsg) return 1 # matched as if the filter does not exist #info(5, "filter return status = " + rest.getStatus()) if (rest.getStatus().find(NGAMS_FAILURE) != -1): match = 1 #info(4, "filter match = " + str(match)) return match
def ngamsGLEAM_Decompress_JobPlugin(srvObj, plugInPars, filename, fileId, fileVersion, diskId): """ srvObj: Reference to NG/AMS Server Object (ngamsServer). plugInPars: Parameters to take into account for the plug-in execution (string).(e.g. scale_factor=4,threshold=1E-5) fileId: File ID for file to test (string). filename: Filename of (complete) (string). fileVersion: Version of file to test (integer). Returns: the return code of the compression plugin (integer). """ pars = "" remove_uc = 0 timeout = 600 # each command should not run more than 10 min, otherwise something is wrong if ((plugInPars != "") and (plugInPars != None)): pars = plugInPars parDic = ngamsPlugInApi.parseRawPlugInPars(pars) if (parDic.has_key('remove_uc')): remove_uc = int(parDic['remove_uc']) if (parDic.has_key('timeout')): timeout = int(parDic['timeout']) if (timeout <= 0): timeout = 600 archive_host = get_contact_ip(srvObj.getCfg()) #cmd1 = "%s -host %s -port 7777 -fileUri %s -cmd QARCHIVE -mimeType application/octet-stream " % (archive_client, archive_host, newfn) cmd2 = "curl http://%s:7777/DISCARD?file_id=%s\\&file_version=%d\\&disk_id=%s\\&execute=1" % ( archive_host, fileId, fileVersion, diskId) """ if (debug): info(3, '*******************************************') info(3, cmd) info(3, cmd1) if (remove_uc): info(3, cmd2) info(3, cmd3) info(3, '*******************************************') return (0, 'Compressed OK') else: """ #re = commands.getstatusoutput(cmd) cmd = "tar xf %s -C %s" % (filename, work_dir) logger.debug("Extracting %s to %s", filename, work_dir) re = execCmd(cmd, timeout) if (0 == re[0]): # archive it back obsId = fileId.split('_')[0] imglist = glob('%s/%s/*.fits' % (work_dir, obsId)) errNo = 0 lasterrMsg = '' for imgfile in imglist: url = 'http://%s:7777/LARCHIVE?fileUri=%s\&mimeType=application/octet-stream\&file_version=%d\&no_versioning=1\&versioning=0' % ( archive_host, imgfile, fileVersion) cmd1 = 'curl --connect-timeout %d %s' % (timeout, url) logger.debug('Local archiving %s', cmd1) re = commands.getstatusoutput(cmd1) if (0 == re[0] and (re[1].count('Successfully handled Archive Pull Request') > 0)): logger.debug( 'Successfully re-archived the untarred FITS file %s', imgfile) else: logger.error( 'Fail to re-archive the untarred FITS file %s: %s', imgfile, re[1]) errNo += 1 lasterrMsg = re[1] #return (re[0], re[1]) if (remove_uc and errNo == 0): # remove the original file if necessary re = execCmd(cmd2, timeout) logger.debug('Removing the tar file %s', filename) if (0 == re[0]): logger.debug('Successfully DISCARDED the tar file %s', filename) else: logger.warning('Fail to DISCARD the tar file %s', filename) # remove the temp file cmd3 = "rm -rf %s/%s" % (work_dir, obsId) logger.debug("Removing the temp directory %s/%s", work_dir, obsId) re = execCmd(cmd3, timeout) if (0 != re[0]): logger.warning('Fail to remove the temp untarred directory %s/%s', work_dir, obsId) if (errNo == 0): return (0, 'Done') else: return (errNo, lasterrMsg.replace("'", "")) else: logger.error('Fail to untar file %s: %s', filename, re[1]) return (re[0], re[1])
def ngamsJob_MITDeliveryPlugin(srvObj, plugInPars, filename, fileId, fileVersion, diskId): """ srvObj: Reference to NG/AMS Server Object (ngamsServer). plugInPars: Parameters to take into account for the plug-in execution (string).(e.g. scale_factor=4,threshold=1E-5) fileId: File ID for file to test (string). filename: Filename of (complete) (string). fileVersion: Version of file to test (integer). Returns: the return code of the compression plugin (integer). """ debug = 0 pars = "" sf = 4 # scaling factor th = 1E-5 # threshold bins = 0 send_crc = 1 to_host = 'eor-10.mit.edu' to_port = '7777' if ((plugInPars != "") and (plugInPars != None)): pars = plugInPars parDic = ngamsPlugInApi.parseRawPlugInPars(pars) if (parDic.has_key('scale_factor')): sf = int(parDic['scale_factor']) if (parDic.has_key('threshold')): th = float(parDic['threshold']) if (parDic.has_key('bins')): bins = int(parDic['bins']) if (parDic.has_key('send_crc')): send_crc = int(parDic['send_crc']) if (parDic.has_key('to_host')): to_host = parDic['to_host'] if (parDic.has_key('to_port')): to_port = parDic['to_port'] if (bins): binstr = '-h %d' % bins else: binstr = '' work_dir = srvObj.getCfg().getRootDirectory() + '/tmp/' newfn = '%s/%s' % (work_dir, os.path.basename(filename)) cmd = "%s -d %d %s %s %s" % (uvcompress, sf, binstr, filename, newfn) cmd3 = "rm %s" % newfn if (os.path.exists(newfn)): execCmd(cmd3) mwaFits = isMWAVisFile(fileId) sendRawFile = False if (mwaFits): re = execCmd(cmd) #if (0 == re[0]): if (os.WIFEXITED(re[0])): # if the child process exit normally (see http://linux.die.net/man/2/wait) if (not bins): retstr = re[1].split('\n')[-1] # just get the elapsed time else: retstr = re[1].split('------ Histogram ------\n')[1] #bbcpurl = "curl --connect-timeout 7200 eor-12.mit.edu:7777/BBCPARC?fileUri=ngas%40146.118.84.67%3A/mnt/mwa01fs/MWA/testfs/output_320M_001.dat\&bport=7790\&bwinsize=%3D32m\&bnum_streams=12\&mimeType=application/octet-stream" bbcpurl = "http://%s:%s/BBCPARC?fileUri=ngas%%40146.118.84.67%%3A%s&bport=7790&bwinsize=%%3D32m&bnum_streams=12&mimeType=application/octet-stream" % (to_host, to_port, newfn) if (send_crc): try: crc32 = getFileCRC(newfn) bbcpurl += "&bchecksum=%s" % str(crc32) except Exception as crcexp: logger.exception('Fail to calculate the file CRC %s', newfn) return (500, str(crcexp).replace("'","")) # send it thru bbcp try: resp = urllib2.urlopen(bbcpurl, timeout = 7200) retstr = resp.read() if (retstr.find("Successfully handled Archive") > -1): logger.debug('Successfully compressed and sent the file %s', newfn) else: logger.error('Fail to sent the file %s: %s', newfn, retstr) return (500, retstr.replace("'","")) except Exception as exp: warn_msg = '' if (type(exp) is urllib2.HTTPError): errors = exp.readlines() for ee in errors: warn_msg += ee warn_msg = 'HTTP error: %s' % (warn_msg) elif (type(exp) is urllib2.URLError): warn_msg = 'Target NGAS server %s is down: %s' % (to_host, str(exp)) else: warn_msg = 'Unexpected error: %s' % (str(exp)) logger.warning('Fail to sent the file %s: %s', newfn, warn_msg) return (500, warn_msg.replace("'","")) finally: # remove the temp file THIS IS DANGEROUS!! re = execCmd(cmd3) #be cautious when copying this line!! if (0 != re[0]): logger.warning('Fail to remove the temp compressed file %s', newfn) return (0, retstr.replace("'","")) else: logger.error('Fail to compress file %s: %s', filename, re[1]) re = execCmd(cmd3) if (0 != re[0]): logger.warning('Fail to remove the temp compressed file %s', newfn) sendRawFile = True if ((not mwaFits) or sendRawFile): #send uncompressed version bbcpurl = "http://%s:%s/BBCPARC?fileUri=ngas%%40146.118.84.67%%3A%s&bport=7790&bwinsize=%%3D32m&bnum_streams=12&mimeType=application/octet-stream" % (to_host, to_port, filename) if (send_crc): try: crc32 = getFileCRC(filename) bbcpurl += "&bchecksum=%s" % str(crc32) except Exception as crcexp: logger.exception('Fail to calculate the file CRC %s', filename) return (500, str(crcexp).replace("'","")) # send it thru bbcp try: resp = urllib2.urlopen(bbcpurl, timeout = 7200) retstr = resp.read() if (retstr.find("Successfully handled Archive") > -1): logger.debug('Successfully sent the uncompressed file %s', filename) return (0, retstr.replace("'","")) else: logger.error('Fail to sent the file %s: %s', filename, retstr) return (500, retstr.replace("'","")) except Exception as exp: warn_msg = '' if (type(exp) is urllib2.HTTPError): errors = exp.readlines() for ee in errors: warn_msg += ee warn_msg = 'HTTP error: %s' % (warn_msg) elif (type(exp) is urllib2.URLError): warn_msg = 'Target NGAS server %s is down: %s' % (to_host, str(exp)) else: warn_msg = 'Unexpected error: %s' % (str(exp)) logger.warning('Fail to sent the file %s: %s', filename, warn_msg) return (500, warn_msg.replace("'",""))
def ngamsTestDppi1(srvObj, reqPropsObj, filename): """ This test DPPI extracts the main header from a FITS file requested from the ESO Archive. Depending on the the DPPI Plug-In Parameters it stores the result in a file or in a buffer in memory. This is made to work on small FITS files. srvObj: Reference to instance of the NG/AMS Server class (ngamsServer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). filename: Name of file to process (string). Returns: DPPI return status object (ngamsDppiStatus). """ statusObj = ngamsDppiStatus.ngamsDppiStatus() # Uncompress the file. procFile, procDir = ngamsPlugInApi.prepProcFile(srvObj.getCfg(), filename) subprocess.check_call(['gunzip', procFile]) procFile = procFile[0:procFile.rfind(".")] # dpallot: fold fails miserably on Mac when dealing with binary files # # Process the output file. #stat, out = commands.getstatusoutput("fold %s" % procFile) #if stat: # raise Exception('Problem while folding %s: %s' % (procFile, out)) head = [] with open(procFile, 'rb') as f: while True: line = f.read(80) assert (line and len(line) == 80) head.append(line) head.append(b'\n') if b'END' in line: break mimeType = "TEST-MIME-TYPE" rawPiPars = srvObj.getCfg().dppi_plugins["ngamsTest.ngamsTestDppi1"].pars cfgParDic = ngamsPlugInApi.parseRawPlugInPars(rawPiPars) head.append(b"\n\nConfiguration Parameters:\n") parList = list(cfgParDic) parList.sort() for par in parList: head.append(six.b("%s=%s\n" % (par, cfgParDic[par]))) head.append(b"\nParameters Transferred:\n") httpParsDic = reqPropsObj.getHttpParsDic() httpPars = list(httpParsDic) httpPars.sort() for httpPar in httpPars: head.append(six.b("%s=%s\n" % (httpPar, httpParsDic[httpPar]))) head.append(b"\nEOF\n") buf = b''.join(head) # Generate status. if (cfgParDic["TARGET"] == "FILE"): outFile = procFile + "_ngamsTestDppi1" with open(outFile, "ab") as fo: fo.write(buf) resObj = ngamsDppiStatus.ngamsDppiResult(NGAMS_PROC_FILE, mimeType, outFile, filename, procDir) else: resObj = ngamsDppiStatus.ngamsDppiResult(NGAMS_PROC_DATA, mimeType, buf, filename) statusObj.addResult(resObj) return statusObj
def ngamsBrotherPT9200DxPlugIn(srvObj, label, reqPropsObj=None): """ Driver for printing labels on the label printer Brother PT-9200DX. srvObj: Reference to instance of the NG/AMS Server class (ngamsServer). label: Label text to print (string). reqPropsObj: NG/AMS request properties object (ngamsReqProps). Returns: Void. """ plugInPars = srvObj.getCfg().getLabelPrinterPlugInPars() logger.info("Executing plug-in ngamsBrotherPT9200DxPlugIn with parameters: " + \ "%s - Label: %s ...", plugInPars, label) parDic = ngamsPlugInApi.parseRawPlugInPars(plugInPars) # Get the font bit pattern dictionary. fontDic = genFontsDictionary(parDic["font_file"]) # Generate the printer control code. printerCode = fontDic["Header"] for i in range(len(label)): if (not fontDic.has_key(label[i])): errMsg = "No font defintion for character: \"" + label[i] +\ "\" - in font definition file: " + parDic["font_file"] +\ " - cannot generate disk label: " + label ngamsPlugInApi.notify(srvObj, NGAMS_NOTIF_ERROR, "ngamsBrotherPT9200DxPlugIn: " +\ "ILLEGAL CHARACTER REQ. FOR PRINTING", errMsg) raise Exception(errMsg) printerCode = printerCode + fontDic[label[i]] printerCode = printerCode + fontDic["Trailer"] # Generate printer file, write the printer control code. tmpDir = ngamsPlugInApi.getTmpDir(srvObj.getCfg()) ngasId = srvObj.getHostId() printerFilename = os.path.normpath(tmpDir + "/ngamsLabel_" + ngasId + ".prn") fo = open(printerFilename, "w") fo.write(printerCode) fo.close() # Write the printer code file to the device. stat, out = ngamsPlugInApi.execCmd("cat " + printerFilename + " > " +\ parDic["dev"]) # This was previously excluded during the "test mode"; a proper way is by # having a configurable parameter if not 'keep_printer_file' in parDic: os.system("rm -f " + printerFilename) if (stat != 0): errMsg = "Problem occurred printing label! Error: " + str(out) ngamsPlugInApi.notify(srvObj, NGAMS_NOTIF_ERROR, "ngamsBrotherPT9200DxPlugIn: " +\ "PROBLEM PRINTING LABEL", errMsg) raise Exception(errMsg)
def ngamsGLEAM_VUW_FilterPI(srvObj, plugInPars, filename, fileId, fileVersion=-1, reqPropsObj=None): """ srvObj: Reference to NG/AMS Server Object (ngamsServer). plugInPars: Parameters to take into account for the plug-in execution (string). fileId: File ID for file to test (string). filename: Filename of (complete) (string). fileVersion: Version of file to test (integer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). Returns: 0 if the file does not match, 1 if it matches the conditions (integer/0|1). """ match = 0 fn, fext = os.path.splitext(fileId) if ( fext.lower() in file_ext and srvObj.getDb().isLastVersion(fileId, fileVersion) ): # only send FITS files, no measurement sets, only send the (known) latest version parDic = [] pars = "" if ((plugInPars != "") and (plugInPars != None)): pars = plugInPars elif (reqPropsObj != None): if (reqPropsObj.hasHttpPar("plug_in_pars")): pars = reqPropsObj.getHttpPar("plug_in_pars") parDic = ngamsPlugInApi.parseRawPlugInPars(pars) if (not parDic.has_key("remote_host") or not parDic.has_key("remote_port")): errMsg = "ngamsGLEAM_VUW_FilterPI: Missing Plug-In Parameter: " +\ "remote_host / remote_port" #raise Exception, errMsg logger.error(errMsg) return 1 # matched as if the remote checking is done host = parDic["remote_host"] sport = parDic["remote_port"] if (not sport.isdigit()): errMsg = "ngamsGLEAM_VUW_FilterPI: Invalid port number: " + sport logger.error(errMsg) return 1 # matched as if the filter does not exist port = int(sport) # Perform the matching. client = ngamsPClient.ngamsPClient(host, port, timeOut=NGAMS_SOCK_TIMEOUT_DEF) try: if (fileVersion == -1): fileVersion = 1 rest = client.get_status(NGAMS_STATUS_CMD, pars=[["file_id", fileId], ["file_version", fileVersion]]) # since the queue will be sorted based on ingestion date, this will ensure the versions are sent by order: # e.g. version1, version2, version3, otherwise, this method will have disordered versions sent if (rest.getStatus().find(NGAMS_FAILURE) != -1): return 1 # matched since file id does not exist #if the same file id (with the latest version) is there already, check CRC """ xmlnode = rest.genXml(genDiskStatus = 1, genFileStatus = 1) tgtCrc = xmlnode.getElementsByTagName('FileStatus')[0].attributes['Checksum'].value cursorObj = srvObj.getDb().getFileInfoList('', fileId, fileVersion) fileList = cursorObj.fetch(1) srcCrc = None for fileInfo in fileList: srcCrc = fileInfo[10] if (cursorObj): del cursorObj if (srcCrc and tgtCrc): if (srcCrc != tgtCrc): match = 1 else: match = 1 # if no CRC information can be found, send the file regardless """ except Exception: errMsg = "Error occurred during checking remote file status " +\ "ngamsGLEAM_VUW_FilterPI" logger.exception(errMsg) return 1 # matched as if the filter does not exist #info(5, "filter return status = " + rest.getStatus()) #info(4, "filter match = " + str(match)) return match