Beispiel #1
0
    def OnTimerExpired(self, event):
        try:
            resets = self.timer.GetNumberOfResets()
            maxResets = int(self.resetsTxtCtrl.GetValue())

            print "Detector reset %d times" % resets

            if (resets >= maxResets):
                print "Number of resets exceeded max"
                emailer = EmailSender(self.emailAddressList)
                emailer.SendEmail(
                    self.senderTxtCtrl.GetValue(), "Scan problem",
                    "The detector was reset too many times.  It requires user intervention",
                    False)
                self.OnStopButtonClick(wx.EVT_IDLE)

        except:
            return

        return
Beispiel #2
0
    def __init__(self, email_list):
        """
        Constructor

        Args:
            :param email_list: a list of email addresses
            :type email_list: list
        """
        self._detectorPrefix = ""
        self._innerScan = ""
        self._outerScan = ""
        self._sender = ""
        self._subject = ""
        self._file_name = ""
        self._mcs_prefix = ""
        self._savedata_prefix = ""
        self._fileCapturePV = None
        self._fileWritePV = None
        self._stopDetectorPV = None
        self._collectModePV = None
        self._scanWaitPV = None
        self._xmapPixelPV = None
        self._mcsPixelPV = None
        self._mcsResetPV = None
        self._usIonChamberPV = None
        self._dsIonChamberPV = None
        self._innerScanCurrentPointPV = None
        self._innerScanNumberPointsPV = None
        self._outerScanCurrentPointPV = None
        self._outerScanNumberPointsPV = None
        self._saveDataMessagePV = None
        self._ringCurrentPV = None
        self._emailList = email_list
        self._mailer = EmailSender(self._emailList)
        
        self._resettingDetector = 0
        self._xmapCurrentPixel = 0
        self._mcsCurrentPixel = 0
        
        return
    def checkServerStart(self):
        """
        check if server started
        """
        time.sleep(30)
        screenlog = self.getScreenInfo()
        while screenlog.find("canceljobmanagertimerentries") == -1:
            #handle assertion
            if screenlog.find("Do you wish to continue")!=-1:
                self.stat = self.FAILED
                self.info ="iserver cannot started, iserver hit assertion"
                if self.logflag == True:
                    self.logger.log(self.info)
                if self.emailflag == True:
                    es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None)
                    es.send()
                return self.FAILED
            #handle crash, need to change##################################################################
            if screenlog.endswith("]#"):
                self.stat = self.FAILED
                self.info ="iserver cannot started, iserver crash"
                if self.logflag == True:
                    self.logger.log(self.info)
                if self.emailflag == True:
                    es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None)
                    es.send()
                return self.FAILED
            #MSTRExec error
            if screenlog.find('MSTRExec Error') != -1:
                print screenlog
                self.stat = self.FAILED
                self.info = "start iserver failed due to MSTRExec error."
                if self.logflag == True:
                    self.logger.log(self.info)
                return self.FAILED
            
            time.sleep(30)
            screenlog = self.getScreenInfo()

        self.stat = self.PASS
        self.info = "server started successfullly"
        if self.logflag == True:
            self.logger.log(self.info)
        return self.PASS  
    def execute(self,State):
        """
        monitor Linux server
        """
        self.init(State)
        if self.logflag == True:
            self.logger.log(self.info)     
        # 1.connect to Linux machine
        self = Command.getClient(self)
        if self.stat == self.FAILED:
            return self.FAILED
        
        # record time
        starttime = time.time()
        duration = 0
        self = Command.getUnixOS(self)
        binpath = Command.checkPath(self.buildInstallPath)+self.buildNo+"/"+self.buildVersion+"/BIN/"+self.OS+"/bin/"
        getStatusCommnad = binpath+"mstrctl -s IntelligenceServer gs"
        
        # 2.monitor
        #check state :./mstrctl -s IntelligenceServer gs
        while duration < int(self.testTimeLimit)*60:
            #1. check if server crash
            #get server status
            statusInfo = Command.executeSsh(getStatusCommnad,self.client)
            xmlInfo = xml.dom.minidom.parseString(statusInfo)
            root=xmlInfo.documentElement
            process_id = ""
            node= root.getElementsByTagName('process_id')[0]
            for node in node.childNodes:
                process_id = node.data
            state = ""
            node= root.getElementsByTagName('state')[0]
            for node in node.childNodes:
                state = node.data
            
            if state.lower().find("stopped") != -1 :
                self.stat = self.FAILED
                self.info ="iserver is stopped, please check if server crashed"
                if self.logflag == True:
                    self.logger.log(self.info)
                if self.emailflag == True:
                    es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None)
                    es.send()
                return self.FAILED
            elif state.lower().find("starting")!= -1:
                self.info ="iserver is starting"
            elif state.lower().find("running")!= -1:
                self.info ="iserver is running fine."
            elif state.lower().find("stopping")!= -1:
                self.stat = self.FAILED
                self.info ="iserver is stopping, please check if server crashed"
                if self.logflag == True:
                    self.logger.log(self.info)
                if self.emailflag == True:
                    es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None)
                    es.send()
                return self.FAILED
            elif state.lower().find("terminated")!= -1:
                self.stat = self.FAILED
                self.info ="iserver is terminated, please check if server crashed"
                if self.logflag == True:
                    self.logger.log(self.info)
                if self.emailflag == True:
                    es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None)
                    es.send()
                return self.FAILED
            elif state.lower().find("pausing")!= -1:
                self.stat = self.FAILED
                self.info ="iserver is pausing, please check if server crashed"
                if self.logflag == True:
                    self.logger.log(self.info)
                if self.emailflag == True:
                    es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None)
                    es.send()
                return self.FAILED
            elif state.lower().find("paused")!= -1:
                self.stat = self.FAILED
                self.info ="iserver is paused, please check if server crashed"
                if self.logflag == True:
                    self.logger.log(self.info)
                if self.emailflag == True:
                    es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None)
                    es.send()
                return self.FAILED
            elif state.lower().find("resuming")!= -1:
                self.info ="iserver is resuming, please check if server crashed" 
            elif state.lower().find("hanging")!= -1:
                self.stat = self.FAILED
                self.info ="iserver is hanging, please check if server crashed"
                if self.logflag == True:
                    self.logger.log(self.info)
                if self.emailflag == True:
                    es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None)
                    es.send()
                return self.FAILED
               
            #2. check if server hit assertion
            screeninfo = self.getScreenInfo()
            if screeninfo.lower().find("assertion") != -1:
                self.stat = self.FAILED
                self.info = "iserver hit assertion"
                if self.logflag == True:
                    self.logger.log(self.info)
                if self.emailflag == True:
                    es = EmailSender(self.emailSender, self.server+"hit assertion:"+screeninfo, self.server+": "+self.info,self.emailToList, "","", attachment=None)
                    es.send()
                return self.FAILED
#             print duration
#             print self.info
#             print state
            time.sleep(int(self.testCheckTime)*60)
            currenttime = time.time()
            duration = int(currenttime-starttime)

        self.stat=self.PASS
        self.info="monitor Linux iserver task has completed!"
        if self.logflag == True:
            self.logger.log(self.info)
        if self.emailflag == True:
            es = EmailSender(self.emailSender, self.server+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None)
            es.send()
        return self.PASS  
        
        
        
Beispiel #5
0
class DetectorErrorHandler(object):
    '''
    classdocs
    '''
    def __init__(self, email_list):
        """
        Constructor

        Args:
            :param email_list: a list of email addresses
            :type email_list: list
        """
        self._detectorPrefix = ""
        self._innerScan = ""
        self._outerScan = ""
        self._sender = ""
        self._subject = ""
        self._file_name = ""
        self._mcs_prefix = ""
        self._savedata_prefix = ""
        self._fileCapturePV = None
        self._fileWritePV = None
        self._stopDetectorPV = None
        self._collectModePV = None
        self._scanWaitPV = None
        self._xmapPixelPV = None
        self._mcsPixelPV = None
        self._mcsResetPV = None
        self._usIonChamberPV = None
        self._dsIonChamberPV = None
        self._innerScanCurrentPointPV = None
        self._innerScanNumberPointsPV = None
        self._outerScanCurrentPointPV = None
        self._outerScanNumberPointsPV = None
        self._saveDataMessagePV = None
        self._ringCurrentPV = None
        self._emailList = email_list
        self._mailer = EmailSender(self._emailList)
        
        self._resettingDetector = 0
        self._xmapCurrentPixel = 0
        self._mcsCurrentPixel = 0
        
        return
        
    def SetDetectorPVs(self, detector_prefix, inner_scan, outer_scan, sender, subject, file_name, mcs_prefix,
                       savedata_prefix, us_ic, ds_ic):
        """"
        Create the PV objects that are used to correct the
        problem with the detector

        Args:
            :param detector_prefix: the ioc prefix for the fluorescence detector PVs
            :type detector_prefix: str
            :param inner_scan: the inner scan record
            :type inner_scan: str
            :param outer_scan: the outer scan record
            :type outer_scan: str
            :param sender: the email address from which to send the update email
            :type sender: str
            :param subject: a subject line for the email
            :type subject: str
            :param file_name: a file to write to which contains the email message
            :type file_name: str
            :param mcs_prefix: the PV prefix for the multi-channel scaler
            :type mcs_prefix: str
            :param savedata_prefix: the PV prefix for saveData
            :type savedata_prefix:str
            :param us_ic: a PV for the value of the upstream ion chamber counts
            :type us_ic: str
            :param ds_ic: a PV for the value of the downstream ion chamber counts
            :type ds_ic: str

        """
        self._detectorPrefix = detector_prefix
        self._innerScan = inner_scan
        self._outerScan = outer_scan
        self._sender = sender
        self._subject = subject
        self._file_name = file_name
        self._mcs_prefix = mcs_prefix
        self._savedata_prefix = savedata_prefix
        
        self._scanWaitPV = PV(self._outerScan + '.WAIT')
        self._usIonChamberPV = PV(us_ic)
        self._dsIonChamberPV = PV(ds_ic)
        self._innerScanCurrentPointPV = PV(self._innerScan + '.CPT')
        self._innerScanNumberPointsPV = PV(self._innerScan + '.NPTS')
        self._outerScanCurrentPointPV = PV(self._outerScan + '.CPT')
        self._outerScanNumberPointsPV = PV(self._outerScan + '.NPTS')
        self._saveDataMessagePV = PV(self._savedata_prefix + "saveData_message")
        self._ringCurrentPV = PV("S:SRcurrentAI")
        self._fileCapturePV = PV(self._detectorPrefix + 'netCDF1:Capture.VAL')
        self._fileWritePV = PV(self._detectorPrefix + 'netCDF1:WriteFile.VAL')
        self._stopDetectorPV = PV(self._detectorPrefix + 'StopAll')
        self._collectModePV = PV(self._detectorPrefix + 'CollectMode')
        self._xmapPixelPV = PV(self._detectorPrefix + "dxp1:CurrentPixel")
        self._mcsPixelPV = PV(self._mcs_prefix + "CurrentChannel")
        self._mcsResetPV = PV(self._mcs_prefix + "StopAll")
        
        return        

    @DelayedEpicsCallback
    def ReportConnectionStatus(self, *args, **kwargs):

        # Initialize some variables
        pvName = kwargs['pvname']  # This is the name of the PV that is running the callback
        pvConnection = kwargs['conn']  # This is the connection status of the PV that is running the callback

        print "%s connection state is %s" % (pvName, pvConnection)
         
        return 
    
    @EpicsFunction
    def ResetDetector(self):
        self._resettingDetector = 1

        self._xmapCurrentPixel = self._xmapPixelPV.get(use_monitor=False)
        self._mcsCurrentPixel = self._mcsPixelPV.get(use_monitor=False)
        
        # Pause the scan.
        print "Pausing scan"
        self._scanWaitPV.put(1, False)
        time.sleep(1)
        # Reset the detector's file plugin and data acquisition
        #print "Stopping file save."
        #self._fileWritePV.put('Done', True, 5)
        print "Stopping data acquisition."
        self._stopDetectorPV.put(1, False)
        print "Stopping data acquisition."
        self._stopDetectorPV.put(1, False)
        print "Stopping data acquisition."
        self._stopDetectorPV.put(1, False)
        time.sleep(1)
        print "Stopping file capture."
        self._fileCapturePV.put(0, False)
        time.sleep(1)
        print "Setting MCS spectra mode."
        self._collectModePV.put(0, False)
        time.sleep(5)
        print "Setting MCA mapping mode."
        self._collectModePV.put(1, False)
        time.sleep(5)

        self._mcsResetPV.put(1, False)

        # Unpause the scan
        print "Resuming scan."
        self._scanWaitPV.put(0, False)

        ret_val = self.WriteStatus(reset=True)
         
        return ret_val

    @EpicsFunction
    def TestResetDetector(self):
        self._resettingDetector = 1
        
        # Pause the scan.
        print "Pausing scan"
        self._scanWaitPV.put(1, False)
        time.sleep(1)
        # Reset the detector's file plugin and data acquisition
        print "Stopping data acquisition."
        self._stopDetectorPV.put(1, False)
        print "Stopping data acquisition."
        self._stopDetectorPV.put(1, False)
        print "Stopping data acquisition."
        self._stopDetectorPV.put(1, False)
        time.sleep(1)
        print "Stopping file capture."
        self._fileCapturePV.put(0, False)
        time.sleep(1)
        print "Setting MCS spectra mode."
        self._collectModePV.put(0, False)
        time.sleep(5)
        print "Setting MCA mapping mode."
        self._collectModePV.put(1, False)
        time.sleep(5)
	
        # Unpause the scan
        print "Resuming scan."
        self._scanWaitPV.put(0, False)
        ret_val = 1
         
        return ret_val
	   
    def WriteStatus(self, reset=False):
        """
        Writes an email that contains the current scan status

        Args:
            :param reset: Is this happening because the detectors was reset?
            :type reset: bool

        """
        try:
            fileHandle = open(self._file_name, 'w+')
            
            currentLine = int(self._outerScanCurrentPointPV.get(use_monitor=False))
            totalLines = int(self._outerScanNumberPointsPV.get(use_monitor=False))
            
            if reset:
                fileHandle.write("Scan error, attempting to reset detector.\n")
            else:
                self._xmapCurrentPixel = self._xmapPixelPV.get(use_monitor=False)
                self._mcsCurrentPixel = self._mcsPixelPV.get(use_monitor=False)
                if (currentLine == totalLines):
                    fileHandle.write("The scan reports that the scan is complete.\n")

            fileHandle.write("Experiment status at time %s \n" % time.strftime("%d %B %Y  %H:%M:%S", time.localtime()))
            fileHandle.write("Synchrotron current: %s \n" % self._ringCurrentPV.get(as_string=True))
            fileHandle.write("Upstream ion chamber reading: %s \n" % self._usIonChamberPV.get(as_string=True))
            fileHandle.write("Downstream ion chamber reading: %s \n" % self._dsIonChamberPV.get(as_string=True))
            fileHandle.write("SaveData message: %s \n" % self._saveDataMessagePV.get(as_string=True))
            fileHandle.write("Current point: %s \n" % self._innerScanCurrentPointPV.get(as_string=True))
            fileHandle.write("Current line: %s \n" % self._outerScanCurrentPointPV.get(as_string=True, use_monitor=False))
            fileHandle.write("Total number of pixels per line: %s \n" % self._innerScanNumberPointsPV.get(as_string=True))
            fileHandle.write("Total number of lines in the scan: %s \n" % self._outerScanNumberPointsPV.get(as_string=True))
            fileHandle.write("Current pixel of MCS: %d\n" % self._mcsCurrentPixel)
            fileHandle.write("Current pixel of XMAP: %d\n" % self._xmapCurrentPixel)
            
            fileHandle.close()
             
            if reset:
                self._mailer.SendEmail(self._sender, "Possible scan error", "",True, self._file_name)
            else:
                if (currentLine == totalLines):
                    self._mailer.SendEmail(self._sender, "Scan record reports completed scan", "", True, self._file_name)
                else:
                    self._mailer.SendEmail(self._sender, self._subject, "", True, self._file_name)
            
            return 1
        except IOError as e:
            for arg in e.args:
                print arg
            return -1
        
    def IsDetectorReset (self):
        return self._resettingDetector
    
    def SetResettingDetector(self, newVal):
        self._resettingDetector = newVal
        return