def OnTimerExpired(self, event):
try:
resets = self.timer.GetNumberOfResets()
maxResets = int(self.resetsTxtCtrl.GetValue())
print "Detector reset %d times" % resets
if (resets >= maxResets):
print "Number of resets exceeded max"
emailer = EmailSender(self.emailAddressList)
emailer.SendEmail(
self.senderTxtCtrl.GetValue(), "Scan problem",
"The detector was reset too many times. It requires user intervention",
False)
self.OnStopButtonClick(wx.EVT_IDLE)
except:
return
return
def __init__(self, email_list):
"""
Constructor
Args:
:param email_list: a list of email addresses
:type email_list: list
"""
self._detectorPrefix = ""
self._innerScan = ""
self._outerScan = ""
self._sender = ""
self._subject = ""
self._file_name = ""
self._mcs_prefix = ""
self._savedata_prefix = ""
self._fileCapturePV = None
self._fileWritePV = None
self._stopDetectorPV = None
self._collectModePV = None
self._scanWaitPV = None
self._xmapPixelPV = None
self._mcsPixelPV = None
self._mcsResetPV = None
self._usIonChamberPV = None
self._dsIonChamberPV = None
self._innerScanCurrentPointPV = None
self._innerScanNumberPointsPV = None
self._outerScanCurrentPointPV = None
self._outerScanNumberPointsPV = None
self._saveDataMessagePV = None
self._ringCurrentPV = None
self._emailList = email_list
self._mailer = EmailSender(self._emailList)
self._resettingDetector = 0
self._xmapCurrentPixel = 0
self._mcsCurrentPixel = 0
return
def checkServerStart(self):
"""
check if server started
"""
time.sleep(30)
screenlog = self.getScreenInfo()
while screenlog.find("canceljobmanagertimerentries") == -1:
#handle assertion
if screenlog.find("Do you wish to continue")!=-1:
self.stat = self.FAILED
self.info ="iserver cannot started, iserver hit assertion"
if self.logflag == True:
self.logger.log(self.info)
if self.emailflag == True:
es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None)
es.send()
return self.FAILED
#handle crash, need to change##################################################################
if screenlog.endswith("]#"):
self.stat = self.FAILED
self.info ="iserver cannot started, iserver crash"
if self.logflag == True:
self.logger.log(self.info)
if self.emailflag == True:
es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None)
es.send()
return self.FAILED
#MSTRExec error
if screenlog.find('MSTRExec Error') != -1:
print screenlog
self.stat = self.FAILED
self.info = "start iserver failed due to MSTRExec error."
if self.logflag == True:
self.logger.log(self.info)
return self.FAILED
time.sleep(30)
screenlog = self.getScreenInfo()
self.stat = self.PASS
self.info = "server started successfullly"
if self.logflag == True:
self.logger.log(self.info)
return self.PASS
def execute(self,State):
"""
monitor Linux server
"""
self.init(State)
if self.logflag == True:
self.logger.log(self.info)
# 1.connect to Linux machine
self = Command.getClient(self)
if self.stat == self.FAILED:
return self.FAILED
# record time
starttime = time.time()
duration = 0
self = Command.getUnixOS(self)
binpath = Command.checkPath(self.buildInstallPath)+self.buildNo+"/"+self.buildVersion+"/BIN/"+self.OS+"/bin/"
getStatusCommnad = binpath+"mstrctl -s IntelligenceServer gs"
# 2.monitor
#check state :./mstrctl -s IntelligenceServer gs
while duration < int(self.testTimeLimit)*60:
#1. check if server crash
#get server status
statusInfo = Command.executeSsh(getStatusCommnad,self.client)
xmlInfo = xml.dom.minidom.parseString(statusInfo)
root=xmlInfo.documentElement
process_id = ""
node= root.getElementsByTagName('process_id')[0]
for node in node.childNodes:
process_id = node.data
state = ""
node= root.getElementsByTagName('state')[0]
for node in node.childNodes:
state = node.data
if state.lower().find("stopped") != -1 :
self.stat = self.FAILED
self.info ="iserver is stopped, please check if server crashed"
if self.logflag == True:
self.logger.log(self.info)
if self.emailflag == True:
es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None)
es.send()
return self.FAILED
elif state.lower().find("starting")!= -1:
self.info ="iserver is starting"
elif state.lower().find("running")!= -1:
self.info ="iserver is running fine."
elif state.lower().find("stopping")!= -1:
self.stat = self.FAILED
self.info ="iserver is stopping, please check if server crashed"
if self.logflag == True:
self.logger.log(self.info)
if self.emailflag == True:
es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None)
es.send()
return self.FAILED
elif state.lower().find("terminated")!= -1:
self.stat = self.FAILED
self.info ="iserver is terminated, please check if server crashed"
if self.logflag == True:
self.logger.log(self.info)
if self.emailflag == True:
es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None)
es.send()
return self.FAILED
elif state.lower().find("pausing")!= -1:
self.stat = self.FAILED
self.info ="iserver is pausing, please check if server crashed"
if self.logflag == True:
self.logger.log(self.info)
if self.emailflag == True:
es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None)
es.send()
return self.FAILED
elif state.lower().find("paused")!= -1:
self.stat = self.FAILED
self.info ="iserver is paused, please check if server crashed"
if self.logflag == True:
self.logger.log(self.info)
if self.emailflag == True:
es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None)
es.send()
return self.FAILED
elif state.lower().find("resuming")!= -1:
self.info ="iserver is resuming, please check if server crashed"
elif state.lower().find("hanging")!= -1:
self.stat = self.FAILED
self.info ="iserver is hanging, please check if server crashed"
if self.logflag == True:
self.logger.log(self.info)
if self.emailflag == True:
es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None)
es.send()
return self.FAILED
#2. check if server hit assertion
screeninfo = self.getScreenInfo()
if screeninfo.lower().find("assertion") != -1:
self.stat = self.FAILED
self.info = "iserver hit assertion"
if self.logflag == True:
self.logger.log(self.info)
if self.emailflag == True:
es = EmailSender(self.emailSender, self.server+"hit assertion:"+screeninfo, self.server+": "+self.info,self.emailToList, "","", attachment=None)
es.send()
return self.FAILED
# print duration
# print self.info
# print state
time.sleep(int(self.testCheckTime)*60)
currenttime = time.time()
duration = int(currenttime-starttime)
self.stat=self.PASS
self.info="monitor Linux iserver task has completed!"
if self.logflag == True:
self.logger.log(self.info)
if self.emailflag == True:
es = EmailSender(self.emailSender, self.server+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None)
es.send()
return self.PASS
class DetectorErrorHandler(object):
'''
classdocs
'''
def __init__(self, email_list):
"""
Constructor
Args:
:param email_list: a list of email addresses
:type email_list: list
"""
self._detectorPrefix = ""
self._innerScan = ""
self._outerScan = ""
self._sender = ""
self._subject = ""
self._file_name = ""
self._mcs_prefix = ""
self._savedata_prefix = ""
self._fileCapturePV = None
self._fileWritePV = None
self._stopDetectorPV = None
self._collectModePV = None
self._scanWaitPV = None
self._xmapPixelPV = None
self._mcsPixelPV = None
self._mcsResetPV = None
self._usIonChamberPV = None
self._dsIonChamberPV = None
self._innerScanCurrentPointPV = None
self._innerScanNumberPointsPV = None
self._outerScanCurrentPointPV = None
self._outerScanNumberPointsPV = None
self._saveDataMessagePV = None
self._ringCurrentPV = None
self._emailList = email_list
self._mailer = EmailSender(self._emailList)
self._resettingDetector = 0
self._xmapCurrentPixel = 0
self._mcsCurrentPixel = 0
return
def SetDetectorPVs(self, detector_prefix, inner_scan, outer_scan, sender, subject, file_name, mcs_prefix,
savedata_prefix, us_ic, ds_ic):
""""
Create the PV objects that are used to correct the
problem with the detector
Args:
:param detector_prefix: the ioc prefix for the fluorescence detector PVs
:type detector_prefix: str
:param inner_scan: the inner scan record
:type inner_scan: str
:param outer_scan: the outer scan record
:type outer_scan: str
:param sender: the email address from which to send the update email
:type sender: str
:param subject: a subject line for the email
:type subject: str
:param file_name: a file to write to which contains the email message
:type file_name: str
:param mcs_prefix: the PV prefix for the multi-channel scaler
:type mcs_prefix: str
:param savedata_prefix: the PV prefix for saveData
:type savedata_prefix:str
:param us_ic: a PV for the value of the upstream ion chamber counts
:type us_ic: str
:param ds_ic: a PV for the value of the downstream ion chamber counts
:type ds_ic: str
"""
self._detectorPrefix = detector_prefix
self._innerScan = inner_scan
self._outerScan = outer_scan
self._sender = sender
self._subject = subject
self._file_name = file_name
self._mcs_prefix = mcs_prefix
self._savedata_prefix = savedata_prefix
self._scanWaitPV = PV(self._outerScan + '.WAIT')
self._usIonChamberPV = PV(us_ic)
self._dsIonChamberPV = PV(ds_ic)
self._innerScanCurrentPointPV = PV(self._innerScan + '.CPT')
self._innerScanNumberPointsPV = PV(self._innerScan + '.NPTS')
self._outerScanCurrentPointPV = PV(self._outerScan + '.CPT')
self._outerScanNumberPointsPV = PV(self._outerScan + '.NPTS')
self._saveDataMessagePV = PV(self._savedata_prefix + "saveData_message")
self._ringCurrentPV = PV("S:SRcurrentAI")
self._fileCapturePV = PV(self._detectorPrefix + 'netCDF1:Capture.VAL')
self._fileWritePV = PV(self._detectorPrefix + 'netCDF1:WriteFile.VAL')
self._stopDetectorPV = PV(self._detectorPrefix + 'StopAll')
self._collectModePV = PV(self._detectorPrefix + 'CollectMode')
self._xmapPixelPV = PV(self._detectorPrefix + "dxp1:CurrentPixel")
self._mcsPixelPV = PV(self._mcs_prefix + "CurrentChannel")
self._mcsResetPV = PV(self._mcs_prefix + "StopAll")
return
@DelayedEpicsCallback
def ReportConnectionStatus(self, *args, **kwargs):
# Initialize some variables
pvName = kwargs['pvname'] # This is the name of the PV that is running the callback
pvConnection = kwargs['conn'] # This is the connection status of the PV that is running the callback
print "%s connection state is %s" % (pvName, pvConnection)
return
@EpicsFunction
def ResetDetector(self):
self._resettingDetector = 1
self._xmapCurrentPixel = self._xmapPixelPV.get(use_monitor=False)
self._mcsCurrentPixel = self._mcsPixelPV.get(use_monitor=False)
# Pause the scan.
print "Pausing scan"
self._scanWaitPV.put(1, False)
time.sleep(1)
# Reset the detector's file plugin and data acquisition
#print "Stopping file save."
#self._fileWritePV.put('Done', True, 5)
print "Stopping data acquisition."
self._stopDetectorPV.put(1, False)
print "Stopping data acquisition."
self._stopDetectorPV.put(1, False)
print "Stopping data acquisition."
self._stopDetectorPV.put(1, False)
time.sleep(1)
print "Stopping file capture."
self._fileCapturePV.put(0, False)
time.sleep(1)
print "Setting MCS spectra mode."
self._collectModePV.put(0, False)
time.sleep(5)
print "Setting MCA mapping mode."
self._collectModePV.put(1, False)
time.sleep(5)
self._mcsResetPV.put(1, False)
# Unpause the scan
print "Resuming scan."
self._scanWaitPV.put(0, False)
ret_val = self.WriteStatus(reset=True)
return ret_val
@EpicsFunction
def TestResetDetector(self):
self._resettingDetector = 1
# Pause the scan.
print "Pausing scan"
self._scanWaitPV.put(1, False)
time.sleep(1)
# Reset the detector's file plugin and data acquisition
print "Stopping data acquisition."
self._stopDetectorPV.put(1, False)
print "Stopping data acquisition."
self._stopDetectorPV.put(1, False)
print "Stopping data acquisition."
self._stopDetectorPV.put(1, False)
time.sleep(1)
print "Stopping file capture."
self._fileCapturePV.put(0, False)
time.sleep(1)
print "Setting MCS spectra mode."
self._collectModePV.put(0, False)
time.sleep(5)
print "Setting MCA mapping mode."
self._collectModePV.put(1, False)
time.sleep(5)
# Unpause the scan
print "Resuming scan."
self._scanWaitPV.put(0, False)
ret_val = 1
return ret_val
def WriteStatus(self, reset=False):
"""
Writes an email that contains the current scan status
Args:
:param reset: Is this happening because the detectors was reset?
:type reset: bool
"""
try:
fileHandle = open(self._file_name, 'w+')
currentLine = int(self._outerScanCurrentPointPV.get(use_monitor=False))
totalLines = int(self._outerScanNumberPointsPV.get(use_monitor=False))
if reset:
fileHandle.write("Scan error, attempting to reset detector.\n")
else:
self._xmapCurrentPixel = self._xmapPixelPV.get(use_monitor=False)
self._mcsCurrentPixel = self._mcsPixelPV.get(use_monitor=False)
if (currentLine == totalLines):
fileHandle.write("The scan reports that the scan is complete.\n")
fileHandle.write("Experiment status at time %s \n" % time.strftime("%d %B %Y %H:%M:%S", time.localtime()))
fileHandle.write("Synchrotron current: %s \n" % self._ringCurrentPV.get(as_string=True))
fileHandle.write("Upstream ion chamber reading: %s \n" % self._usIonChamberPV.get(as_string=True))
fileHandle.write("Downstream ion chamber reading: %s \n" % self._dsIonChamberPV.get(as_string=True))
fileHandle.write("SaveData message: %s \n" % self._saveDataMessagePV.get(as_string=True))
fileHandle.write("Current point: %s \n" % self._innerScanCurrentPointPV.get(as_string=True))
fileHandle.write("Current line: %s \n" % self._outerScanCurrentPointPV.get(as_string=True, use_monitor=False))
fileHandle.write("Total number of pixels per line: %s \n" % self._innerScanNumberPointsPV.get(as_string=True))
fileHandle.write("Total number of lines in the scan: %s \n" % self._outerScanNumberPointsPV.get(as_string=True))
fileHandle.write("Current pixel of MCS: %d\n" % self._mcsCurrentPixel)
fileHandle.write("Current pixel of XMAP: %d\n" % self._xmapCurrentPixel)
fileHandle.close()
if reset:
self._mailer.SendEmail(self._sender, "Possible scan error", "",True, self._file_name)
else:
if (currentLine == totalLines):
self._mailer.SendEmail(self._sender, "Scan record reports completed scan", "", True, self._file_name)
else:
self._mailer.SendEmail(self._sender, self._subject, "", True, self._file_name)
return 1
except IOError as e:
for arg in e.args:
print arg
return -1
def IsDetectorReset (self):
return self._resettingDetector
def SetResettingDetector(self, newVal):
self._resettingDetector = newVal
return