def checkTarball(filename): """ Check that the tarball is correct filename: Name of tarball file (string). Returns: Void. """ err = 0 tmpFilename = filename.replace(":", "_") + "_tmp" try: execCmd("ln -s " + filename + " " + tmpFilename) stat, out, _ = execCmd("tar -tf " + tmpFilename) rmFile(tmpFilename) if (stat != 0): errMsg = str(out).replace("\n", " --- ") err = 1 except Exception as e: rmFile(tmpFilename) errMsg = str(e) err = 1 if (err): errMsg = "Error checking tarball: " + errMsg errMsg = genLog("NGAMS_ER_DAPI_BAD_FILE", [filename, "ngasTarBallPlugIn", errMsg]) raise Exception(errMsg)
def ngamsSuspensionPlugIn(srvObj): """ Suspension Plug-In to suspend an NGAS host. srvObj: Reference to instance of the NG/AMS Server (ngamsServer). Returns: Void. """ execCmd("sudo /sbin/shutdown -h now")
def run(pars, srvObj, filename): user = pars['user'] host = pars['host'] tgt_dir = pars['tgt_dir'] tgt_filename = os.path.join(tgt_dir, os.path.basename(filename)) tgt = '%s@%s:%s' % (user, host, tgt_filename) execCmd(['scp', filename, tgt], timeOut=300, shell=False) logger.info("Successfully scp'd logfile %s to %s", filename, tgt)
def ngamsWakeUpPlugIn(srvObj, hostId): """ Wake-Up Plug-In to wake up a suspended NGAS host. srvObj: Reference to instance of the NG/AMS Server (ngamsServer). hostId: Name of NGAS host to be woken up (string). Returns: Void. """ hostDic = ngamsHighLevelLib.\ getHostInfoFromHostIds(srvObj.getDb(), [hostId]) if hostId not in hostDic: errMsg = "ngamsWakeUpPlugIn: Could not wake up host: " + hostId +\ " - host not defined in NGAS DB." raise Exception(errMsg) networkDevs = srvObj.getCfg().getWakeUpPlugInPars() cmdFormat = "sudo /sbin/ether-wake -i %s -b " +\ hostDic[hostId].getMacAddress() logger.debug("Waking up suspended host: %s", hostId) for dev in networkDevs.split(","): cmd = cmdFormat % dev logger.debug("Broadcasting wake-up package - command: %s", cmd) stat, out, _ = execCmd(cmd) if (stat != 0): format = "ngamsWakeUpPlugIn: Problem waking up host: %s " +\ ". Error: %s." errMsg = format % (hostId, str(out).replace("\n", " ")) raise Exception(errMsg)
def mountToMountpoint(devName, mntPt, readOnly, fstype="reiserfs"): """ Method mounts a device <devName> to a mount point <mntPt>. The optional parameter fstype can be used to specify a filesystem type different than reiserfs. To mount a device which is in the /etc/fstab use the following command: mountToMountpoint("", <mntPt>, fstype="") Synopsis: mountToMountpoint(devName, mntPt[, fstype = <fstype>]) devName: Device name (string). mntPt: Mount point (string). readOnly: 0 if not read-only, 1 if read-only (integer). fstype: File system type (string). Returns: Void. """ command = ["sudo", "mount"] if (readOnly): command.append('-r') command += [devName, mntPt] logger.debug("Command to mount disk: %s", subprocess.list2cmdline(command)) # if (getMountedDev(mntPt)): # already mounted # warnMsg = "Device " + getMountedDev(mntPt) + ' already mounted ' + \ # 'to specified mountpoint: ' + mntPt # info(1,warnMsg) # else: # if not os.path.exists(mntPt): # try: # posix.mkdir(mntPt) # except exceptions.OSError,e: # errMsg = "Failed creating mountpoint " + mntPt + ":" + str(e) # error(errMsg) # raise Exception, errMsg # stat, out = commands.getstatusoutput(command) # if (stat != 0): # errMsg = "Failed mounting device " + getMountedDev(mntPt) + \ # ':' + out # raise Exception, errMsg if not os.path.exists(mntPt): try: os.mkdir(mntPt) except OSError as e: errMsg = "Failed creating mountpoint " + mntPt + ":" + str(e) logger.exception(errMsg) raise stat, out, _ = execCmd(command) if stat != 0 or b'already mounted' in out: errMsg = "Failed mounting device. Device/mount point: %s/%s" %\ (devName, mntPt) raise Exception(errMsg)
def handle_event(evt): if evt.file_version != 1: logger.info('File %s arrived with version != 1 (=%d), skipping', evt.file_id, evt.file_version) return if evt.file_id not in self.file_ids: logger.info( 'File %s not in list of files to be source-extracted, skipping', evt._file_id) return cmd = [x.replace('%s', evt.file_id) for x in self.command] logger.info('Executing command to trigger source extraction: %s', subprocess.list2cmdline(cmd)) ngamsCore.execCmd(cmd)
def rmMod(module): """ Wrapper around the rmmod program. NOTE: This is Linux specific and won't work on any other system. Synopsis: rmMod(<module>) module: Module name (string). Returns: 0 upon success or if module is not loaded (int). """ stat = checkModule(module) if (stat[1]): execCmd("/sbin/rmmod " + module) else: return 0 return 0
def exportCont(contId): """ Export all units found on the controller. The function does not complain if errors occur. contId: ID (number) of the controller (integer). Returns: Void. """ return # We don't want this to be executed ... # Unit UnitType Status %Cmpl Stripe Size(GB) Cache AVerify IgnECC # ------------------------------------------------------------------------------ # u0 RAID-5 OK - 256K 1862.59 ON OFF OFF # u1 RAID-5 OK - 256K 1490.07 ON OFF OFF # u2 SPARE OK - - 372.603 - OFF - # Port Status Unit Size Blocks Serial # --------------------------------------------------------------- # p0 OK u0 372.61 GB 781422768 KRFS02RAGWWHKC # p1 OK u0 372.61 GB 781422768 KRFT06RAGW9ZAC # p2 OK u0 372.61 GB 781422768 KRFS26RAGWB7ZC # ... cmd = "sudo /usr/local/sbin/tw_cli info c%s" % str(contId) stat, out, _ = execCmd(cmd) # Just look for lines starting with "u". unitList = [] for line in out.split("\n"): line = line.strip() if (line == ""): continue if (line[0] == "u"): unitList.append(line.split(" ")[0]) for unit in unitList: cmd = "sudo /usr/local/sbin/tw_cli /c%d/%s export quiet" %\ (int(contId), unit) logger.debug("Invoking command to export 3ware unit: %s ...", cmd) stat, out, _ = execCmd(cmd) logger.debug("Result of command: %s to export 3ware unit: %d", cmd, stat)
def rescanCont(contId): """ Rescan a 3ware controller. contId: ID (number) of the controller (integer). Returns: Void. """ return # Don't want to execute this cmd = "sudo /usr/local/sbin/tw_cli /c%d rescan" % (int(contId)) logger.debug("Invoking command to rescan 3ware unit: %s ...", cmd) stat, out, _ = execCmd(cmd) logger.debug("Result of command: %s to rescan 3ware unit: %d", cmd, stat)
def checkModule(module): """ Check whether module <module> exists and is loaded. Method returns a tuple, <mtup>, where mtab[0] reports the existence of a module, and mtab[1] reports the load status of a module. NOTE: This is Linux specific and won't work on any other system Synopsis: checkModule(<module>) module: Module name (string). Returns: 2-element tuple (tuple). """ mex, ml = (-1, -1) # default return code if type(module) != type(""): errMsg = "Parameter module passed to " + __name__ + "." + \ "checkModule has invalid type (should be string)" raise Exception(errMsg) # command = "/sbin/modprobe -l | /bin/grep \"/" + module + ".o$\"" command = "find /lib/modules -name " + module + ".*" stat, out, _ = execCmd(command) if stat > 0 or not out.strip(): mex, ml = (0, 0) # there is no module like this else: mex = 1 # module exists stat, out, _ = execCmd("/sbin/lsmod | /bin/grep " + \ "\"^" + module + "\"") if out: ml = 1 # module is loaded else: ml = 0 # module not loaded mtup = (mex, ml) return mtup
def umountMountpoint(mntPt): """ Method unmounts a device mounted on <mntPt>. Synopsis: unmountMountpoint(mntPt) mntPoint: Mount point (string). Returns: Void. """ command = ["sudo", "umount", mntPt] stat, out, _ = execCmd(command) if stat != 0 or b'not mounted' in out: errMsg = "Failed unmounting. Mount point: %s" % mntPt raise Exception(errMsg)
def _execCClient(unpackXmlStat = 1, pars = []): """ Execute the NG/AMS C-Client on the shell. cmdLineParsDic: Dictionary with command line parameters (dictionary). pars: Extra parameters for invoking NGAMS C-Client (list). Returns: List with status objects or stdout output from c-client: [<stat obj>, ...] or <stdout c-client> (list|string). """ cmd = ['ngamsCClient'] for opt, val in pars: cmd.append(opt) cmd.append(str(val)) if '-servers' not in (x for x, _ in pars): cmd.append('-host') cmd.append('127.0.0.1') cmd.append('-status') env = os.environ.copy() env['NGAMS_VERBOSE_LEVEL'] = '0' _, out, _ = execCmd(cmd, shell=False) out = utils.b2s(out) if (unpackXmlStat): statObjList = [] xmlStatList = out.split("Command repetition counter:") for xmlStat in xmlStatList: # Clean up the output. xmlStat = xmlStat.strip() if (xmlStat == ""): continue idx = 0 while (xmlStat[idx] != "<"): idx += 1 xmlStat = xmlStat[idx:] # Unpack it. statObj = ngamsStatus.ngamsStatus().unpackXmlDoc(xmlStat) statObjList.append(statObj) out = statObjList return out
def getContInfo(contList): """ Get the dump about each controller, concatenate this and return it. contList: List with controllers in the system (list/integer). Returns: Accumulated ASCII output of info about controllers (string). """ out = "" for contId in contList: cmd = "sudo /usr/local/sbin/tw_cli info c%s" % str(contId) logger.debug("Executing 3ware client tool: %s", cmd) stat, outTmp, _ = execCmd(cmd) logger.debug("Executed 3ware client tool. Status: %d", stat) if (stat): raise Exception("Error invoking 3ware Command Line Tool: " +\ str(out)) else: out += outTmp out += "\n" return out
def ngamsTileCompress(filename): """ Tile compress the referenced file. filename: Filename (string). Returns: Void. """ tmpFilename = filename + ".tmp" try: comprCmd = "imcopy %s '%s[compress]'" % (filename, tmpFilename) logger.debug("Command to tile compress file: %s", comprCmd) stat, out, _ = execCmd(comprCmd) if (stat != 0): msg = "Error compressing file: %s. Error: %s" %\ (filename, stat.replace("\n", " ")) raise Exception(msg) mvFile(tmpFilename, filename) logger.debug("Successfully tile compressed file: %s", filename) except: rmFile(tmpFilename) raise
def getControllers(): """ Query the controllers available, and return this in a list. Returns: List with available 3ware controllers (list). """ cmd = "sudo /usr/local/sbin/tw_cli info" stat, out, _ = execCmd(cmd) if (stat): raise Exception("Error invoking 3ware Command Line Tool: " + str(out)) contList = [] for line in out.split("\n"): line = line.strip() if (line): if (line.find("Controller") == 0): # "Controller 1: 8506-4LP (4)" contNo = line.split(" ")[1].split(":")[0] contList.append(contNo) elif (line[0] == "c"): contNo = int(line.split(" ")[0][1:]) contList.append(contNo) contList.sort() return contList
def ngasExtractFitsHdrDppi(srvObj, reqPropsObj, filename): """ This DPPI extracts the main header from a FITS file requested from the ESO Archive. srvObj: Reference to instance of the NG/AMS Server class (ngamsServer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). filename: Name of file to process (string). Returns: DPPI return status object (ngamsDppiStatus). Side effect: This DPPI works directly on the archived file, since it is read-only access. SPECIFIC DOCUMENTATION: This DPPI controls the call to the printhead module. If the Example URL (single line): http://ngasdev1:7777/RETRIEVE ?file_id=MIDI.2004-02-11T04:16:04.528& processing=ngasExtractFitsHdrDppi& processing_pars=header%3D99 The header parameter is optional, if not specified the primary header is returned. Valid values for the header parameter are numbers between 0 and 99. If numbers are specified which are either outside the range or bigger than the number of headers (including the primary) the primary header will be returned. Headers are counted from 0 starting with the primary header. 99 is a special value as it returns all headers concatenated in a single file. If 'xml=vo' is specified headers are returned using a slightly modified VOTable (XML) format. If 'xml=xfits' is specified headers are returned using the XFits (XML) format. struct=1 returns the structure of the FITS file. tsv=1 returns the headers in a tab separated format suitable for direct ingest into the header repository. """ logger.debug("Entering ngasExtractFitsHdrDppi() ...") statusObj = ngamsDppiStatus.ngamsDppiStatus() if (reqPropsObj.hasHttpPar("processing_pars")): pars = ngamsPlugInApi.\ parseRawPlugInPars(reqPropsObj.getHttpPar("processing_pars")) else: # default is to extract the primary header pars = {'header': 0} logger.debug("ngasExtractFitsHdrDppi: %s %r", filename, pars) PARS = set(['header', 'xml', 'skey', 'struct', 'tsv', 'check']) # initial settings for printhead xtract = 0 parse = 0 xmlVals = ['xfits', 'vo'] xmlfl = '' skeyfl = 0 skey = 'END' show = 0 struct = 0 tsv = 0 check = 0 mergefl = 0 hfl = 0 mode = 1 result = '' err = '' ext = 'hdr' if pars.has_key('header'): # extract a certain header: if value == 99 all headers are extracted, # for any other value that header is extracted. headers are # counted from 0 hfl = 1 struct = 0 show = pars['header'] try: head = int(show) except: err = "ngasExtractFitsHdrDppi: Invalid type for header " +\ "parameter. Should be int" if head < 0 or head > 99: err = "ngasExtractFitsHdrDppi: Invalid value specified for " +\ "header parameter." if pars.has_key('xml'): # if this key exists we do a conversion to XFits XML. struct = 0 if pars['xml'] in xmlVals: xmlfl = pars['xml'] else: err = "ngasExtractFitsHdrDppi: Invalid value for xml " +\ "parameter. Should be 'vo|xfits': "+ pars['xml'] ext = 'xml' if pars.has_key('skey'): # extract just one keyword. CAUTION: No checking done! skey = pars['skey'].strip() skeyfl = 1 keyParts = skey.split() ext = 'txt' head = int(head) if head < 0: head = 0 if ((not re.match('[a-zA-Z]', skey[0])) or (len(keyParts) > 1 and keyParts[0] != 'HIERARCH') or (len(keyParts[0]) > 8)): err = "ngasExtractFitsHdrDppi: Invalid value for skey " +\ "parameter specified. Must be a valid FITS keyword:",\ skey if pars.has_key('struct'): # return only the structure of the FITS file. Value of the # parameter is ignored head = -99 struct = 1 ext = 'txt' if pars.has_key('tsv'): # extract header in tsv format. Parameter value is ignored struct = 1 tsv = 1 ext = 'txt' head = int(head) if head < 0: head = 0 if pars.has_key('check'): # head structure and calculate the checksum of the data part. head = -99 struct = 1 check = 1 # printhead supports a list of files, but here we only use one fils = [filename] base = os.path.basename(filename) pos = base.rfind('.fits') file_id = base[:pos] if len(set(pars) - PARS) != 0: # detect unsupported parameters xpars = set(pars) - PARS err = "ngasExtractFitsHdrDppi: Unsupported option(s): %s\n" + \ "Valid options are:\n" + \ "header=<number> where number is an integer between 0 " +\ "and max(extension)-1 or 99 (for all)\n" +\ "xml=<format>, where format is [vo|xfits]\n" +\ "struct=1\n" +\ "skey=<FITS keyword>, should be a valid keyword, crude " +\ "checking is done\n" + \ "tsv=1\n\n" + \ "combinations are allowed and should be separated by " +\ "a comma character.\n" err = err % xpars ext = 'txt' file_id = 'error' result = '' # initialize result string if err != '': result = err if result == '': for f in fils: cmd = constructCommand(f, head, struct, skey, tsv, xmlfl, mode, check) logger.debug('Executing command: %s', cmd) stat, result, _ = execCmd(cmd) if stat != 0: errMsg = "Processing of header for file %s failed: %s" % ( filename, result) raise Exception(errMsg) resFilename = file_id + "." + ext try: # mime-type guessing does not work sometimes, we force it in that case. mimeType = ngamsPlugInApi.determineMimeType(srvObj.getCfg(), resFilename) except: if ext == 'xml': mimeType = 'text/xml' else: mimeType = 'text/ascii' resObj = ngamsDppiStatus.ngamsDppiResult(NGAMS_PROC_DATA, mimeType, result, resFilename, '') statusObj.addResult(resObj) logger.debug("Leaving ngasExtractFitsHdrDppi() ...") return statusObj
def _registerExec(srvObj, fileListDbmName, tmpFilePat, diskInfoDic, reqPropsObj=None): """ Register the files listed in the File List DBM (ngamsDbm), which match the mime-type(s) either specified in the 'mimeType' parameter, or if this is not specified, which match all the mime-types specified in the configuration file. When the registration procedure has been executed, the function sends an Email Notification message indicating which files were registered if the HTTP parameter 'notif_email' is given. The functions creates a File Info Objects per file handled and writes this in a temporary file, which is a DBM file. The keys in this DB is simply the file number in the sequence of files handled, pointing to a pickled ngamsFileInfo object. Each of the File Info Objects indicates if the file was registered or not. This is done by setting the tag of the File Info Object to one of the following values: REGISTERED: The file was successfully registered in the NGAS DB FAILED: The file was selected for registration but could not be properly registered because of inconsistencies. The status will be of the format: 'FAILED[: <reason>]'. REJECTED: A file found under the specified path directory was not accepted for cloning, usually because the mime-type was not correct. The status will be of the format: 'REJECTED[: <reason>]'. Note, that registration is comparable to archiving of files. For that reason a DAPI must be provided for each type of file that should be registered. If this is not fullfilled, the file registration will fail. Only files stored on one of the NGAS disks configured in the configuration file, are considered. Files stored in other locations are ignored. srvObj: Instance of NG/AMS Server object (ngamsServer). fileListDbmName: Name of a DBM containing the information about the files to be registered. Each element in the list is referred to by a key, which is a number. These points to a pickled list for each file containing the following information: [<Filename>, <Disk ID>, <Mime-Type>] The information for each disk concerned is also contained in the DB referred to by its Disk ID and by its mount point. The data is a pickled instance of the ngamsDiskInfo class (string). tmpFilePat: Pattern for temporary files used during the registration process (string). diskInfoDic: Dictionary with Disk IDs as keys pointing to the info about the disk (dictionary/ngamsDiskInfo). reqPropsObj: If an NG/AMS Request Properties Object is given, the Request Status will be updated as the request is carried out (ngamsReqProps). Returns: Void. """ emailNotif = 0 if (reqPropsObj): if (reqPropsObj.hasHttpPar("notif_email")): emailNotif = 1 # Create the temporary BSD DB to contain the information for the # Email Notification Message. if (emailNotif): regDbmName = tmpFilePat + "_NOTIF_EMAIL" regDbm = ngamsDbm.ngamsDbm(regDbmName, writePerm=1) # Open the DBM containing the list of files to (possibly) register. fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm=1) # Want to parse files in alphabetical order. # TODO: Portatibility issue. Try to avoid UNIX shell commands for sorting. tmpFileList = tmpFilePat + "_FILE_LIST" rmFile(tmpFileList) with open(tmpFileList, "wb") as fo: fileListDbm.initKeyPtr() while (1): dbmKey, fileInfo = fileListDbm.getNext() if (not dbmKey): break fo.write(dbmKey + b"\n") sortFileList = tmpFilePat + "_SORT_FILE_LIST" rmFile(sortFileList) shellCmd = "sort %s > %s" % (tmpFileList, sortFileList) stat, out, err = ngamsCore.execCmd(shellCmd) if (stat != 0): raise Exception("Error executing command: %s. Error: %s, %s" %\ (shellCmd, str(out), str(err))) rmFile(tmpFileList) # Go through each file in the list, check if the mime-type is among the # ones, which apply for registration. If yes try to register the file # by invoking the corresponding DAPI on the file. fileRegCount = 0 fileFailCount = 0 fileRejectCount = 0 regTimeAccu = 0.0 fileCount = 0 fo = open(sortFileList) run = 1 while (run): reg_start = time.time() dbmKey = fo.readline() if (dbmKey.strip() == ""): run = 0 continue fileInfo = fileListDbm.get(dbmKey[0:-1]) filename = fileInfo[0] diskId = fileInfo[1] mimeType = fileInfo[2] # Register the file. Check first, that exactly this file is # not already registered. In case it is, the file will be rejected. regPi = srvObj.getCfg().register_plugins[mimeType] logger.debug("Plugin found for %s: %s", mimeType, regPi) params = ngamsPlugInApi.parseRawPlugInPars(regPi.pars) tmpReqPropsObj = ngamsReqProps.ngamsReqProps().\ setMimeType(mimeType).\ setStagingFilename(filename).\ setTargDiskInfo(diskInfoDic[diskId]).\ setHttpMethod(NGAMS_HTTP_GET).\ setCmd(NGAMS_REGISTER_CMD).\ setSize(os.path.getsize(filename)).\ setFileUri(filename).\ setNoReplication(1) tmpFileObj = ngamsFileInfo.ngamsFileInfo() try: # Invoke Registration Plug-In. piName = regPi.name plugInMethod = loadPlugInEntryPoint(piName) piRes = plugInMethod(srvObj, tmpReqPropsObj, params) del tmpReqPropsObj # Check if this file is already registered on this disk. In case # yes, it is not registered again. files = srvObj.db.getFileSummary1(srvObj.getHostId(), [piRes.getDiskId()], [piRes.getFileId()]) fileRegistered = 0 for tmpFileInfo in files: tmpMtPt = tmpFileInfo[ngamsDbCore.SUM1_MT_PT] tmpFilename = tmpFileInfo[ngamsDbCore.SUM1_FILENAME] tmpComplFilename = os.path.normpath(tmpMtPt + "/" +\ tmpFilename) if (tmpComplFilename == filename): fileRegistered = 1 break if (fileRegistered): fileRejectCount += 1 tmpMsgForm = "REJECTED: File with File ID/Version: %s/%d " +\ "and path: %s is already registered on disk " +\ "with Disk ID: %s" tmpMsg = tmpMsgForm % (piRes.getFileId(), piRes.getFileVersion(), filename, piRes.getDiskId()) logger.warning(tmpMsg + ". File is not registered again.") if (emailNotif): tmpFileObj.\ setDiskId(diskId).setFilename(filename).\ setTag(tmpMsg) regDbm.addIncKey(tmpFileObj) if (reqPropsObj): reqPropsObj.incActualCount(1) ngamsHighLevelLib.stdReqTimeStatUpdate( srvObj, reqPropsObj, regTimeAccu) regTimeAccu += time.time() - reg_start fileCount += 1 continue # Calculate checksum. We maintain the old name for backwards # compatibility crc_variant = srvObj.cfg.getCRCVariant() if crc_variant == ngamsFileUtils.CHECKSUM_CRC32_INCONSISTENT: crc_variant = 'ngamsGenCrc32' checksum = ngamsFileUtils.get_checksum(65536, filename, crc_variant) or '' # Move file and update information about file in the NGAS DB. mvFile(filename, piRes.getCompleteFilename()) ngamsArchiveUtils.updateFileInfoDb(srvObj, piRes, checksum, crc_variant) ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), piRes) ngamsLib.makeFileReadOnly(piRes.getCompleteFilename()) if (emailNotif): uncomprSize = piRes.getUncomprSize() ingestDate = time.time() creDateSecs = getFileCreationTime(filename) tmpFileObj.\ setDiskId(diskId).\ setFilename(filename).\ setFileId(piRes.getFileId()).\ setFileVersion(piRes.getFileVersion()).\ setFormat(piRes.getFormat()).\ setFileSize(piRes.getFileSize()).\ setUncompressedFileSize(uncomprSize).\ setCompression(piRes.getCompression()).\ setIngestionDate(ingestDate).\ setIgnore(0).\ setChecksum(checksum).\ setChecksumPlugIn(crc_variant).\ setFileStatus(NGAMS_FILE_STATUS_OK).\ setCreationDate(creDateSecs).\ setTag("REGISTERED") fileRegCount += 1 # If running as a cache archive, update the Cache New Files DBM # with the information about the new file. if (srvObj.getCachingActive()): fileVer = fio.getFileVersion() ngamsCacheControlThread.addEntryNewFilesDbm( srvObj, diskId, piRes.getFileId(), fileVer, filename) # Generate a confirmation log entry. msg = genLog("NGAMS_INFO_FILE_REGISTERED", [ filename, piRes.getFileId(), piRes.getFileVersion(), piRes.getFormat() ]) time.sleep(0.005) regTime = time.time() - reg_start msg = msg + ". Time: %.3fs." % (regTime) logger.info(msg, extra={'to_syslog': 1}) except Exception as e: errMsg = genLog("NGAMS_ER_FILE_REG_FAILED", [filename, str(e)]) logger.error(errMsg) if (emailNotif): tmpFileObj.\ setDiskId(diskId).setFilename(filename).\ setTag(errMsg) fileFailCount += 1 regTime = time.time() - reg_start # TODO (rtobar, 2016-01): Why don't we raise an exception here? # Otherwise the command appears as successful on the # client-side # Add the file information in the registration report. if (emailNotif): regDbm.addIncKey(tmpFileObj) # Update request status time information. regTimeAccu += regTime if (reqPropsObj): reqPropsObj.incActualCount(1) ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj, regTimeAccu) fileCount += 1 fo.close() rmFile(sortFileList) if (emailNotif): regDbm.sync() del fileListDbm rmFile(fileListDbmName + "*") # Final update of the Request Status. if (reqPropsObj): if (reqPropsObj.getExpectedCount() and reqPropsObj.getActualCount()): complPercent = (100.0 * (float(reqPropsObj.getActualCount()) / float(reqPropsObj.getExpectedCount()))) else: complPercent = 100.0 reqPropsObj.setCompletionPercent(complPercent, 1) reqPropsObj.setCompletionTime(1) srvObj.updateRequestDb(reqPropsObj) # Send Register Report with list of files cloned to a possible # requestor(select) of this. if (emailNotif): xmlStat = 0 if (xmlStat): # Create an instance of the File List Class, used to store the # Registration Report. regStat = ngamsFileList.\ ngamsFileList("FILE_REGISTRATION_STATUS", "Status report for file " +\ "registration") # Loop over the file objects in the BSD DB and add these # in the status object. regDbm.initKeyPtr() while (1): key, tmpFileObj = regDbm.getNext() if (not key): break regStat.addFileInfoObj(tmpFileObj) # Set overall status of registration procedure. regStat.setStatus("Files Found: " + str(fileCount + 1) + ", "+\ "Files Registered: " + str(fileRegCount) +\ ", " +\ "Files Failed: " + str(fileFailCount) + ", " +\ "Files Rejected: " + str(fileRejectCount)) status = srvObj.genStatus(NGAMS_SUCCESS, "REGISTER command status report").\ addFileList(regStat) statRep = status.genXmlDoc() statRep = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, statRep) mimeType = NGAMS_XML_MT else: # Generate a 'simple' ASCII report. statRep = tmpFilePat + "_NOTIF_EMAIL.txt" fo = open(statRep, "w") if (reqPropsObj): path = reqPropsObj.getHttpPar("path") else: path = "-----" if (fileCount): timePerFile = (regTimeAccu / fileCount) else: timePerFile = 0 tmpFormat = "REGISTER STATUS REPORT:\n\n" +\ "==Summary:\n\n" +\ "Date: %s\n" +\ "NGAS Host: %s\n" +\ "Search Path: %s\n" +\ "Total Number of Files: %d\n" +\ "Number of Registered Files: %d\n" +\ "Number of Failed Files: %d\n" +\ "Number of Rejected Files: %d\n" +\ "Total processing time (s): %.3f\n" +\ "Handling time per file (s): %.3f\n\n" +\ "==File List:\n\n" fo.write(tmpFormat % (toiso8601(), srvObj.getHostId(), path, fileCount, fileRegCount, fileFailCount, fileRejectCount, regTimeAccu, timePerFile)) tmpFormat = "%-80s %-32s %-3s %-10s\n" fo.write(tmpFormat %\ ("Filename", "File ID", "Ver", "Status")) fo.write(tmpFormat % (80 * "-", 32 * "-", 3 * "-", 10 * "-")) regDbm.initKeyPtr() while (1): key, tmpFileObj = regDbm.getNext() if (not key): break mtPt = diskInfoDic[tmpFileObj.getDiskId()].getMountPoint() filename = os.path.normpath(mtPt + "/" +\ tmpFileObj.getFilename()) line = tmpFormat %\ (filename, tmpFileObj.getFileId(), str(tmpFileObj.getFileVersion()), tmpFileObj.getTag()) fo.write(line) fo.write(128 * "-") fo.write("\n\n==END\n") fo.close() mimeType = NGAMS_TEXT_MT # Send out the status report. emailAdrList = reqPropsObj.getHttpPar("notif_email").split(",") attachmentName = "RegisterStatusReport" if (reqPropsObj.hasHttpPar("path")): attachmentName += "-" + reqPropsObj.getHttpPar("path").\ replace("/", "_") ngamsNotification.notify(srvObj.host_id, srvObj.cfg, NGAMS_NOTIF_INFO, "REGISTER STATUS REPORT", statRep, recList=emailAdrList, force=1, contentType=mimeType, attachmentName=attachmentName) del regDbm rmFile(regDbmName + "*") rmFile(statRep) # Generate final status log + exit. if (fileCount > 0): timePerFile = (regTimeAccu / fileCount) else: timePerFile = 0.0 msg = "Registration procedure finished processing Register Request - " + \ "terminating. Files handled: %d. Total time: %.3fs. " + \ "Average time per file: %.3fs." logger.debug(msg, fileCount, regTimeAccu, timePerFile)
def runAllTests(notifEmail=None, skip=None): """ Run all tests in ngasUtils/test. notifEmail: List of email recipients that should be notified about the test results (list/email addresses). skip: Test Suites or Test Cases to skip. Comma separated list (string). Returns: Void. """ skipDic = {} if (skip): for test in skip.split(","): skipDic[test] = 1 testModList = getTestList() startTime = time.time() failModDic = {} noOfTests = len(testModList) testCount = 0 format = "Running Test Suite: %-32s %-8s" line = "\nNGAS UTILITIES FUNCTIONAL TESTS - TEST REPORT\n" print line testRep = line + "\n" line = "Date: %s" % PccUtTime.TimeStamp().getTimeStamp() print line testRep += line + "\n" line = "Host: %s" % getHostName() print line testRep += line + "\n" line = "NG/AMS Version: %s\n" % getNgamsVersion() print line testRep += line + "\n" for mod in testModList: if (skipDic.has_key(mod)): continue testCount += 1 line = format % (mod, ("(#%d/%d)" % (testCount, noOfTests))) testRep += line sys.stdout.write(line) sys.stdout.flush() suiteStartTime = time.time() stat, stdout, stderr = ngamsCore.execCmd("python " + mod + ".py", NGAMS_TEST_MAX_TS_TIME) testTime = (time.time() - suiteStartTime) if (testTime >= NGAMS_TEST_MAX_TS_TIME): failModDic[mod] = "TIME-OUT" stat = " - %-5.1fs - TIME-OUT!!\n" % testTime elif (stdout.find("FAILED") != -1): failModDic[mod] = stdout + " --- " + stderr stat = " - %-5.1fs - FAILURE!!\n" % testTime else: stat = " - %-5.1fs - SUCCESS\n" % testTime sys.stdout.write(stat) sys.stdout.flush() testRep += stat execTime = (time.time() - startTime) line = "\n\nExecuted %d Test Suites in %.3f seconds\n\n" %\ (len(testModList), execTime) print line testRep += line if (failModDic): line = 80 * "=" print line testRep += line line = "\n\nFAILED TEST SUITES:\n\n" print line testRep += line failMods = failModDic.keys() failMods.sort() for failMod in failMods: line = "\n%s:\n%s\n" % (failMod, failModDic[failMod]) print line testRep += line + "\n" line = 80 * "-" print line testRep += line line = 80 * "=" print line testRep += line # Send out email with test report if requested. if (notifEmail): notifEmail = notifEmail.split(",") par = "NgamsCfg.Server[1].MountRootDirectory" ngamsCfgObj = ngamsConfig.ngamsConfig().storeVal(par, "/NGAS") ngamsHighLevelLib.sendEmail(ngamsCfgObj, "localhost", "NGAS UTILITIES FUNCTIONAL TESTS REPORT", notifEmail, "ngas@%s" %\ ngamsLib.getCompleteHostName(), testRep)
def parseGen2Controller(rootMtPt, contList, devStartIdx, ngasStartIdx, contOut): """ The output of the 2nd generation 3ware Command Line Utility used for NGAS look like this: Unit UnitType Status %Cmpl Stripe Size(GB) Cache AVerify IgnECC --------------------------------------------------------------------------- Port Status Unit Size Blocks Serial --------------------------------------------------------------- p0 NOT-PRESENT - - - - p1 OK - 372.61 GB 781422768 KRFS06RAGGXTAC p2 NOT-PRESENT - - - - p3 OK - 372.61 GB 781422768 KRFS06RAGBND5C p4 OK - 372.61 GB 781422768 KRFS06RAGGWD7C p5 OK - 372.61 GB 781422768 KRFS06RAGGXTAC p6 OK - 372.61 GB 781422768 KRFS06RAGGXUWC p7 OK - 372.61 GB 781422768 KRFS06RAGBND5C Unit UnitType Status %Cmpl Stripe Size(GB) Cache AVerify IgnECC --------------------------------------------------------------------------- u0 RAID-1 OK - - 65.1826 ON OFF OFF u1 RAID-1 OK - - 372.519 ON OFF OFF Port Status Unit Size Blocks Serial --------------------------------------------------------------- p0 OK u0 69.25 GB 145226112 WD-WMAKE1317691 p1 OK u0 69.25 GB 145226112 WD-WMAKE1317543 p2 OK u1 372.61 GB 781422768 KRFS06RAGBNB1C p3 OK u1 372.61 GB 781422768 KRFS06RAGBGP0C rootMtPt: Root mount point (string). contList: List with controllers in the system (list/integer). devStartIdx: Start index of disk array (string/a|b|c...). ngasStartIdx: Startindex of first disk to be used by NGAS (string/a|b|c...) contOut: Output from 3ware controller (string). Returns: Dictionary with info about the disks (dictionary). """ # Parse the output from the 3ware Command Line Tool. diskInfoDic = {} slotCount = 0 # Not used for the moment. contCount = 0 # Controller counter. tmpPorts = 0 # Number of ports current controller. portsSum = 0 # Total sum of ports parsed. devIdx = ord(devStartIdx) if ngasStartIdx < devStartIdx: ngasStartIdx = devStartIdx idx = 0 bufLines = contOut.split("\n") bufLen = len(bufLines) while (idx < bufLen): line = bufLines[idx].strip() logger.debug("Parsing 3ware status line: %s", line) # Next controller? if ((line.find("Unit ") != -1) and (line.find("UnitType ") != -1)): unitTypeDic = {} contNo = contList[contCount] contCount += 1 portsSum += tmpPorts tmpPorts = 0 # Look for 1st unit - check if a JBOD or RAID unit. raid = 0 while (idx < bufLen): idx += 1 line = bufLines[idx].strip() if (len(line)): if ((line.find("Port") != -1) and (line.find("Status") != -1)): break elif (line[0].strip() == "u"): cfg = filter(None, line.split(" ")) if (cfg[1].find("RAID") != -1): unitTypeDic[cfg[0]] = "RAID" else: unitTypeDic[cfg[0]] = "JBOD" # Get the info for each JBOD disk. Parse until the start of the # next controller status or till the end of the status output. idx += 1 while (idx < bufLen): line = bufLines[idx].strip() logger.debug("Parsing 3ware status line: %s", line) if (line == ""): idx += 1 continue if ((line.find("Unit ") != -1) and (line.find("UnitType ") != -1)): break # Break to outer loop # Take the line if it contains disk info. if (line[0] == "p"): tmpPorts += 1 diskInfo = filter(None, line.split(" ")) if ((diskInfo[1] != "NOT-PRESENT") and (unitTypeDic[diskInfo[2]] == "JBOD")): # A disk is found, get the info. portNo = int(diskInfo[0][1:]) slotId = str((((contCount - 1) > 0) * portsSum) +\ (portNo + 1)) # Get status, model ID, serial number and capacity. cmd = "sudo /usr/local/sbin/tw_cli info c%d p%d " +\ "status model serial capacity" cmd = cmd % (int(contNo), int(portNo)) stat, out, _ = execCmd(cmd) outLines = out.split("\n") status = filter(None, outLines[0].split(" "))[3] model = filter(None, outLines[1].split(" "))[3] serialNo = filter(None, outLines[2].split(" "))[3] capGb = int(float(filter(None, outLines[3].\ split(" "))[3]) + 0.5) diskType = "MAGNETIC DISK/ATA" if (model.find("HDS") == 0): manufact = "Hitachi Data Systems" elif (model.find("WDC") == 0): manufact = "Western Digital Corporation" else: manufact = "UNKNOWN" diskId = model + "-" + serialNo mtPt = rootMtPt + "/volume" + slotId devName = "/dev/sd" + chr(devIdx +\ len(diskInfoDic)) + "1" diskInfoDic[str(slotId)] = ngamsPhysDiskInfo.\ ngamsPhysDiskInfo().\ setPortNo(portNo).\ setSlotId(slotId).\ setMountPoint(mtPt).\ setStatus(status).\ setCapacityGb(capGb).\ setModel(model).\ setSerialNo(serialNo).\ setType(diskType).\ setManufacturer(manufact).\ setDiskId(diskId).\ setDeviceName(devName) idx += 1 idx -= 1 # Decrease count to avoid index exceeding buffer length. idx += 1 # Main loop buffer index # AWI: added to fix ATF problem [AW] diskInfoDicUsed = {} for di in diskInfoDic: if diskInfoDic[di].getDeviceName() >= '/dev/sd' + ngasStartIdx + '1': diskInfoDicUsed.update({di: diskInfoDic[di]}) # AWI return diskInfoDicUsed
def parseHtmlInfo(url, rootMtPt, slotIds=["*"]): """ Query the information about the disks as an HTML page, and parse the page and return the information in the format as defined for the NG/AMS Disk Dictionary. url: URL from where to query the disk status (string). rootMtPt: Root mount point to be used (string) slotIds: List containing the Slot IDs to take into account. If given as '*' all slot are considered (list/string). Returns: Dictionary containing objects which are instances of the class ngamsPhysDiskInfo (dictionary). """ # Check if the 3ware WEB server can be accessed. cmd = "wget -T 2 -t 1 %s" % url stat, _, _ = execCmd(cmd) if (stat): logger.warning("Problem contacting local 3ware WEB server via URL: %s", url) return {} fd = urllib.urlopen(url) #fd = open(url) buf = fd.read() fd.close() lines = buf.split("\n") idx = 0 length = len(lines) takeAllSlots = 0 if (len(slotIds)): if (slotIds[0] == '*'): takeAllSlots = 1 diskInfoDic = {} while (idx < length): if "tech_drive_header\">Port" in lines[idx]: # We expect the following output: # # <tr><td class="tech_drive_header">Port 0</td></tr> # <tr><td class="tech_drive_info"> # <table> # <tr><td class="tech_unit_field">Status:</td>\ # <td class="tech_unit_value">\ # <font color="#3333FF">OK</font></td></tr> # <tr><td class="tech_unit_field">Capacity:\ # </td><td class="tech_unit_value">\ # 38.34 GB (80418240 blocks)</td></tr> # <tr><td class="tech_unit_field">\ # Model:</td><td class="tech_unit_value">\ # IBM-DTLA-305040 </td></tr> # <tr><td class="tech_unit_field">Serial number:\ # </td><td class="tech_unit_value"> YJ0YJ070913</td>\ # </tr> # <tr><td class="tech_unit_field">Unit number:</td>\ # <td class="tech_unit_value">0</td></tr> # <li>Physical drive (port) 7 # <ul> # <li>Status: <font color="#3333FF">OK</font> # <li>Capacity: 41.17 GB (80418240 blocks) # <li>Model: IBM-DTLA-305040 # <li>Serial number: YJ0YJ070913 # <li>Unit number: 7 portNo = int(lines[idx].split(" ")[2].split("<")[0]) # Check if this slot should be taken into account, i.e., if the # current Slot ID is in the list given as input to the function. slotId = str(int(portNo) + 1) if ((not takeAllSlots) and slotId not in slotIds): idx = idx + 1 continue idx = idx + 3 status = lines[idx].split(">")[5].split("<")[0] idx = idx + 1 capacityGb = float(lines[idx].split(">")[4].split(" ")[0]) capacityBlocks = int( lines[idx].split(">")[4].split("(")[1].split(" ")[0]) idx = idx + 1 model = lines[idx].split(">")[4].split(" ")[0] diskType = "MAGNETIC DISK/ATA" manufacturer = model.split("-")[0] if (manufacturer[0:2] == "IC"): manufacturer = "IBM" idx = idx + 1 serialNo = lines[idx].split(">")[4].split("<")[0].strip() idx = idx + 1 unitNo = int(lines[idx].split(">")[4].split("<")[0]) diskId = model + "-" + serialNo mtPt = rootMtPt + "/data" + str(slotId) deviceName = "/dev/sd" + chr(97 + len(diskInfoDic)) + '1' diskInfoDic[str(slotId)] = ngamsPhysDiskInfo.ngamsPhysDiskInfo().\ setPortNo(portNo).\ setSlotId(slotId).\ setMountPoint(mtPt).\ setStatus(status).\ setCapacityGb(capacityGb).\ setModel(model).\ setSerialNo(serialNo).\ setType(diskType).\ setManufacturer(manufacturer).\ setDiskId(diskId).\ setDeviceName(deviceName) idx = idx + 1 return diskInfoDic