def OnTimerExpired(self, event): try: resets = self.timer.GetNumberOfResets() maxResets = int(self.resetsTxtCtrl.GetValue()) print "Detector reset %d times" % resets if (resets >= maxResets): print "Number of resets exceeded max" emailer = EmailSender(self.emailAddressList) emailer.SendEmail( self.senderTxtCtrl.GetValue(), "Scan problem", "The detector was reset too many times. It requires user intervention", False) self.OnStopButtonClick(wx.EVT_IDLE) except: return return
def __init__(self, email_list): """ Constructor Args: :param email_list: a list of email addresses :type email_list: list """ self._detectorPrefix = "" self._innerScan = "" self._outerScan = "" self._sender = "" self._subject = "" self._file_name = "" self._mcs_prefix = "" self._savedata_prefix = "" self._fileCapturePV = None self._fileWritePV = None self._stopDetectorPV = None self._collectModePV = None self._scanWaitPV = None self._xmapPixelPV = None self._mcsPixelPV = None self._mcsResetPV = None self._usIonChamberPV = None self._dsIonChamberPV = None self._innerScanCurrentPointPV = None self._innerScanNumberPointsPV = None self._outerScanCurrentPointPV = None self._outerScanNumberPointsPV = None self._saveDataMessagePV = None self._ringCurrentPV = None self._emailList = email_list self._mailer = EmailSender(self._emailList) self._resettingDetector = 0 self._xmapCurrentPixel = 0 self._mcsCurrentPixel = 0 return
def checkServerStart(self): """ check if server started """ time.sleep(30) screenlog = self.getScreenInfo() while screenlog.find("canceljobmanagertimerentries") == -1: #handle assertion if screenlog.find("Do you wish to continue")!=-1: self.stat = self.FAILED self.info ="iserver cannot started, iserver hit assertion" if self.logflag == True: self.logger.log(self.info) if self.emailflag == True: es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None) es.send() return self.FAILED #handle crash, need to change################################################################## if screenlog.endswith("]#"): self.stat = self.FAILED self.info ="iserver cannot started, iserver crash" if self.logflag == True: self.logger.log(self.info) if self.emailflag == True: es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None) es.send() return self.FAILED #MSTRExec error if screenlog.find('MSTRExec Error') != -1: print screenlog self.stat = self.FAILED self.info = "start iserver failed due to MSTRExec error." if self.logflag == True: self.logger.log(self.info) return self.FAILED time.sleep(30) screenlog = self.getScreenInfo() self.stat = self.PASS self.info = "server started successfullly" if self.logflag == True: self.logger.log(self.info) return self.PASS
def execute(self,State): """ monitor Linux server """ self.init(State) if self.logflag == True: self.logger.log(self.info) # 1.connect to Linux machine self = Command.getClient(self) if self.stat == self.FAILED: return self.FAILED # record time starttime = time.time() duration = 0 self = Command.getUnixOS(self) binpath = Command.checkPath(self.buildInstallPath)+self.buildNo+"/"+self.buildVersion+"/BIN/"+self.OS+"/bin/" getStatusCommnad = binpath+"mstrctl -s IntelligenceServer gs" # 2.monitor #check state :./mstrctl -s IntelligenceServer gs while duration < int(self.testTimeLimit)*60: #1. check if server crash #get server status statusInfo = Command.executeSsh(getStatusCommnad,self.client) xmlInfo = xml.dom.minidom.parseString(statusInfo) root=xmlInfo.documentElement process_id = "" node= root.getElementsByTagName('process_id')[0] for node in node.childNodes: process_id = node.data state = "" node= root.getElementsByTagName('state')[0] for node in node.childNodes: state = node.data if state.lower().find("stopped") != -1 : self.stat = self.FAILED self.info ="iserver is stopped, please check if server crashed" if self.logflag == True: self.logger.log(self.info) if self.emailflag == True: es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None) es.send() return self.FAILED elif state.lower().find("starting")!= -1: self.info ="iserver is starting" elif state.lower().find("running")!= -1: self.info ="iserver is running fine." elif state.lower().find("stopping")!= -1: self.stat = self.FAILED self.info ="iserver is stopping, please check if server crashed" if self.logflag == True: self.logger.log(self.info) if self.emailflag == True: es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None) es.send() return self.FAILED elif state.lower().find("terminated")!= -1: self.stat = self.FAILED self.info ="iserver is terminated, please check if server crashed" if self.logflag == True: self.logger.log(self.info) if self.emailflag == True: es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None) es.send() return self.FAILED elif state.lower().find("pausing")!= -1: self.stat = self.FAILED self.info ="iserver is pausing, please check if server crashed" if self.logflag == True: self.logger.log(self.info) if self.emailflag == True: es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None) es.send() return self.FAILED elif state.lower().find("paused")!= -1: self.stat = self.FAILED self.info ="iserver is paused, please check if server crashed" if self.logflag == True: self.logger.log(self.info) if self.emailflag == True: es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None) es.send() return self.FAILED elif state.lower().find("resuming")!= -1: self.info ="iserver is resuming, please check if server crashed" elif state.lower().find("hanging")!= -1: self.stat = self.FAILED self.info ="iserver is hanging, please check if server crashed" if self.logflag == True: self.logger.log(self.info) if self.emailflag == True: es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None) es.send() return self.FAILED #2. check if server hit assertion screeninfo = self.getScreenInfo() if screeninfo.lower().find("assertion") != -1: self.stat = self.FAILED self.info = "iserver hit assertion" if self.logflag == True: self.logger.log(self.info) if self.emailflag == True: es = EmailSender(self.emailSender, self.server+"hit assertion:"+screeninfo, self.server+": "+self.info,self.emailToList, "","", attachment=None) es.send() return self.FAILED # print duration # print self.info # print state time.sleep(int(self.testCheckTime)*60) currenttime = time.time() duration = int(currenttime-starttime) self.stat=self.PASS self.info="monitor Linux iserver task has completed!" if self.logflag == True: self.logger.log(self.info) if self.emailflag == True: es = EmailSender(self.emailSender, self.server+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None) es.send() return self.PASS
class DetectorErrorHandler(object): ''' classdocs ''' def __init__(self, email_list): """ Constructor Args: :param email_list: a list of email addresses :type email_list: list """ self._detectorPrefix = "" self._innerScan = "" self._outerScan = "" self._sender = "" self._subject = "" self._file_name = "" self._mcs_prefix = "" self._savedata_prefix = "" self._fileCapturePV = None self._fileWritePV = None self._stopDetectorPV = None self._collectModePV = None self._scanWaitPV = None self._xmapPixelPV = None self._mcsPixelPV = None self._mcsResetPV = None self._usIonChamberPV = None self._dsIonChamberPV = None self._innerScanCurrentPointPV = None self._innerScanNumberPointsPV = None self._outerScanCurrentPointPV = None self._outerScanNumberPointsPV = None self._saveDataMessagePV = None self._ringCurrentPV = None self._emailList = email_list self._mailer = EmailSender(self._emailList) self._resettingDetector = 0 self._xmapCurrentPixel = 0 self._mcsCurrentPixel = 0 return def SetDetectorPVs(self, detector_prefix, inner_scan, outer_scan, sender, subject, file_name, mcs_prefix, savedata_prefix, us_ic, ds_ic): """" Create the PV objects that are used to correct the problem with the detector Args: :param detector_prefix: the ioc prefix for the fluorescence detector PVs :type detector_prefix: str :param inner_scan: the inner scan record :type inner_scan: str :param outer_scan: the outer scan record :type outer_scan: str :param sender: the email address from which to send the update email :type sender: str :param subject: a subject line for the email :type subject: str :param file_name: a file to write to which contains the email message :type file_name: str :param mcs_prefix: the PV prefix for the multi-channel scaler :type mcs_prefix: str :param savedata_prefix: the PV prefix for saveData :type savedata_prefix:str :param us_ic: a PV for the value of the upstream ion chamber counts :type us_ic: str :param ds_ic: a PV for the value of the downstream ion chamber counts :type ds_ic: str """ self._detectorPrefix = detector_prefix self._innerScan = inner_scan self._outerScan = outer_scan self._sender = sender self._subject = subject self._file_name = file_name self._mcs_prefix = mcs_prefix self._savedata_prefix = savedata_prefix self._scanWaitPV = PV(self._outerScan + '.WAIT') self._usIonChamberPV = PV(us_ic) self._dsIonChamberPV = PV(ds_ic) self._innerScanCurrentPointPV = PV(self._innerScan + '.CPT') self._innerScanNumberPointsPV = PV(self._innerScan + '.NPTS') self._outerScanCurrentPointPV = PV(self._outerScan + '.CPT') self._outerScanNumberPointsPV = PV(self._outerScan + '.NPTS') self._saveDataMessagePV = PV(self._savedata_prefix + "saveData_message") self._ringCurrentPV = PV("S:SRcurrentAI") self._fileCapturePV = PV(self._detectorPrefix + 'netCDF1:Capture.VAL') self._fileWritePV = PV(self._detectorPrefix + 'netCDF1:WriteFile.VAL') self._stopDetectorPV = PV(self._detectorPrefix + 'StopAll') self._collectModePV = PV(self._detectorPrefix + 'CollectMode') self._xmapPixelPV = PV(self._detectorPrefix + "dxp1:CurrentPixel") self._mcsPixelPV = PV(self._mcs_prefix + "CurrentChannel") self._mcsResetPV = PV(self._mcs_prefix + "StopAll") return @DelayedEpicsCallback def ReportConnectionStatus(self, *args, **kwargs): # Initialize some variables pvName = kwargs['pvname'] # This is the name of the PV that is running the callback pvConnection = kwargs['conn'] # This is the connection status of the PV that is running the callback print "%s connection state is %s" % (pvName, pvConnection) return @EpicsFunction def ResetDetector(self): self._resettingDetector = 1 self._xmapCurrentPixel = self._xmapPixelPV.get(use_monitor=False) self._mcsCurrentPixel = self._mcsPixelPV.get(use_monitor=False) # Pause the scan. print "Pausing scan" self._scanWaitPV.put(1, False) time.sleep(1) # Reset the detector's file plugin and data acquisition #print "Stopping file save." #self._fileWritePV.put('Done', True, 5) print "Stopping data acquisition." self._stopDetectorPV.put(1, False) print "Stopping data acquisition." self._stopDetectorPV.put(1, False) print "Stopping data acquisition." self._stopDetectorPV.put(1, False) time.sleep(1) print "Stopping file capture." self._fileCapturePV.put(0, False) time.sleep(1) print "Setting MCS spectra mode." self._collectModePV.put(0, False) time.sleep(5) print "Setting MCA mapping mode." self._collectModePV.put(1, False) time.sleep(5) self._mcsResetPV.put(1, False) # Unpause the scan print "Resuming scan." self._scanWaitPV.put(0, False) ret_val = self.WriteStatus(reset=True) return ret_val @EpicsFunction def TestResetDetector(self): self._resettingDetector = 1 # Pause the scan. print "Pausing scan" self._scanWaitPV.put(1, False) time.sleep(1) # Reset the detector's file plugin and data acquisition print "Stopping data acquisition." self._stopDetectorPV.put(1, False) print "Stopping data acquisition." self._stopDetectorPV.put(1, False) print "Stopping data acquisition." self._stopDetectorPV.put(1, False) time.sleep(1) print "Stopping file capture." self._fileCapturePV.put(0, False) time.sleep(1) print "Setting MCS spectra mode." self._collectModePV.put(0, False) time.sleep(5) print "Setting MCA mapping mode." self._collectModePV.put(1, False) time.sleep(5) # Unpause the scan print "Resuming scan." self._scanWaitPV.put(0, False) ret_val = 1 return ret_val def WriteStatus(self, reset=False): """ Writes an email that contains the current scan status Args: :param reset: Is this happening because the detectors was reset? :type reset: bool """ try: fileHandle = open(self._file_name, 'w+') currentLine = int(self._outerScanCurrentPointPV.get(use_monitor=False)) totalLines = int(self._outerScanNumberPointsPV.get(use_monitor=False)) if reset: fileHandle.write("Scan error, attempting to reset detector.\n") else: self._xmapCurrentPixel = self._xmapPixelPV.get(use_monitor=False) self._mcsCurrentPixel = self._mcsPixelPV.get(use_monitor=False) if (currentLine == totalLines): fileHandle.write("The scan reports that the scan is complete.\n") fileHandle.write("Experiment status at time %s \n" % time.strftime("%d %B %Y %H:%M:%S", time.localtime())) fileHandle.write("Synchrotron current: %s \n" % self._ringCurrentPV.get(as_string=True)) fileHandle.write("Upstream ion chamber reading: %s \n" % self._usIonChamberPV.get(as_string=True)) fileHandle.write("Downstream ion chamber reading: %s \n" % self._dsIonChamberPV.get(as_string=True)) fileHandle.write("SaveData message: %s \n" % self._saveDataMessagePV.get(as_string=True)) fileHandle.write("Current point: %s \n" % self._innerScanCurrentPointPV.get(as_string=True)) fileHandle.write("Current line: %s \n" % self._outerScanCurrentPointPV.get(as_string=True, use_monitor=False)) fileHandle.write("Total number of pixels per line: %s \n" % self._innerScanNumberPointsPV.get(as_string=True)) fileHandle.write("Total number of lines in the scan: %s \n" % self._outerScanNumberPointsPV.get(as_string=True)) fileHandle.write("Current pixel of MCS: %d\n" % self._mcsCurrentPixel) fileHandle.write("Current pixel of XMAP: %d\n" % self._xmapCurrentPixel) fileHandle.close() if reset: self._mailer.SendEmail(self._sender, "Possible scan error", "",True, self._file_name) else: if (currentLine == totalLines): self._mailer.SendEmail(self._sender, "Scan record reports completed scan", "", True, self._file_name) else: self._mailer.SendEmail(self._sender, self._subject, "", True, self._file_name) return 1 except IOError as e: for arg in e.args: print arg return -1 def IsDetectorReset (self): return self._resettingDetector def SetResettingDetector(self, newVal): self._resettingDetector = newVal return