def handleDetectorTimeout(self, id):
"""this function is triggered if a Detector timesout"""
self.publishQueue.put((id,
stateObject([
CommonStates.ConnectionProblem,
CommonStates.ConnectionProblem
])))
def waitForUpdates(self):
"""wait for updates from unmapped detectors"""
while True:
try:
message = self.socketDetectorUpdates.recv()
message = json.loads(message.decode())
id = message["id"]
if id not in self.detectors:
self.log("received message with unknown id: %s" % id, True)
self.socketDetectorUpdates.send(codes.idUnknown)
continue
self.socketDetectorUpdates.send(codes.ok)
det = self.detectors[id]
state = message["state"]
configTag = None
comment = None
if "tag" in message:
configTag = message["tag"]
if "comment" in message:
comment = message["comment"]
self.publishQueue.put((det.id,
stateObject([
det.getMappedStateForState(state),
state, configTag, comment
])))
except zmq.error.ContextTerminated:
self.socketDetectorUpdates.close()
break
def globalSystemTimeout(self, id):
"""this function is triggered if a global System timesout"""
self.publishQueue.put((id,
stateObject([
CommonStates.ConnectionProblem,
CommonStates.ConnectionProblem
])))
def transition(self, command, nopublish=False):
"""try to transition the global Statemachine"""
oldstate = self.stateMachine.currentState
if self.stateMachine.transition(command):
if not nopublish:
self.publishQueue.put(
(self.id,
stateObject([
self.stateMachine.currentState,
self.stateMachine.currentState, self.globalTag, None
])))
self.log("Global Statechange: " + oldstate + " -> " +
self.stateMachine.currentState)
def waitForUpdates(self):
"""wait for updates from Detectors"""
while True:
try:
message = self.socketDetectorUpdates.recv()
message = json.loads(message.decode())
id = message["id"]
if id in self.detectors:
subSystemObject = self.detectors[id]
elif id in self.globalSystems:
subSystemObject = self.globalSystems[id]
else:
self.log("received message with unknown id: %s" % id, True)
self.socketDetectorUpdates.send(codes.idUnknown)
continue
self.socketDetectorUpdates.send(codes.ok)
if not subSystemObject.checkSequence(
message["sequenceNumber"]):
#update is obsolete
continue
state = message["state"]
configTag = None
comment = None
if "tag" in message:
configTag = message["tag"]
if "comment" in message:
comment = message["comment"]
if (subSystemObject.stateMachine.currentState == state):
pass
#continue
self.publishQueue.put((
id,
stateObject([
subSystemObject.getMappedStateForState(state), state,
configTag, comment
]),
))
except zmq.error.ContextTerminated:
self.socketDetectorUpdates.close()
break
def ping(self):
"""sends periodic heartbeats to the subsystem"""
pingSocket = self.createPingSocket()
while True:
try:
pingSocket.send(codes.ping)
pingSocket.recv()
if self.connected != True:
self.logfunction("%s is connected" % self.name)
ret = self.getStateFromSystem()
if not ret:
#sometimes when PCA and DC start both at once there is a timeout from getting state(maybe the socket isn't ready idk)
continue
state, configTag = ret
self.connected = True
self.reconnectFunction(
stateObject([
self.getMappedStateForState(state), state,
configTag, None
]))
except zmq.Again:
#timeout
if self.connected == True or self.connected == None:
self.connected = False
self.logfunction("timeout pinging %s" % self.name, True)
self.timeoutFunction()
#reset socket
pingSocket.close()
pingSocket = self.createPingSocket()
except zmq.error.ContextTerminated:
#process is terminating; end loop
break
except zmq.error.ZMQError:
#unexpected error
#reset socket
pingSocket.close()
pingSocket = self.createPingSocket()
time.sleep(self.pingInterval)
def receive_status(socket, pcaid=None):
"""receive status from PCA retuns None on error"""
if pcaid:
errorString = " receiving status for %s" % pcaid
else:
errorString = " receiving status"
try:
id, sequence, state = socket.recv_multipart()
except zmq.Again:
print("timeout" + errorString)
return None
except Exception as e:
print("error" + errorString)
return None
if id != codes.done:
id = id.decode()
state = stateObject(json.loads(state.decode()))
else:
state = None
sequence = intFromBytes(sequence)
return [id, sequence, state]
def handleDetectorTimeout(self, id):
"""handler function for ping timeout"""
self.publishQueue.put((id, stateObject("Connection Problem")))
def __init__(self, id):
self.id = id
#for time measurement of configuring all Systems
self.start_time = 0.0
self.end_time = 0.0
self.detector_configure_time_start = 0.0
#create lock
try:
self.lock = zc.lockfile.LockFile('/tmp/lock' + self.id,
content_template='{pid}')
except zc.lockfile.LockError:
print("other Process is already Running " + self.id)
exit(1)
self.terminate = threading.Event()
self.initdone = threading.Event()
#handle SIGTERM signal
signal.signal(signal.SIGTERM, self.terminatePCA)
#read config File
config = configparser.ConfigParser()
config.read("init.cfg")
conf = config["Default"]
self.context = zmq.Context()
self.receive_timeout = int(conf['receive_timeout'])
self.ECSAdress = conf['ECAAddress']
self.ECARequestPort = conf['ECARequestPort']
def checkConsistencyRequest(detectorList):
#currently unused
detectorList = DataObjectCollection(json.loads(detectorList),
detectorDataObject)
start_new_thread(self.checkSystemConsistency, (detectorList, ))
return codes.ok
def measureConfigureTime(arg=None):
"""just for timemeasure purpose use self.configure instead"""
self.start_time = time.time()
return self.configure(arg)
#lookup table for recieved commands
self.functionForCodeDictionary = {
#codes.getReady: self.configure,
codes.getReady: measureConfigureTime,
codes.start: self.startRecording,
codes.stop: self.stopRecording,
codes.removeDetector: self.removeDetector,
codes.addDetector: self.addDetector,
codes.abort: self.abort,
codes.check: checkConsistencyRequest,
codes.lock: self.lockPartition,
codes.unlock: self.unlockPartition,
codes.reset: self.resetSystem,
codes.subsystemMessage: self.handleSystemMessage
}
#get your config
configECS = None
while configECS == None:
requestSocket = self.context.socket(zmq.REQ)
requestSocket.connect("tcp://%s:%s" %
(conf['ECAAddress'], conf['ECARequestPort']))
requestSocket.setsockopt(zmq.RCVTIMEO,
int(conf['receive_timeout']))
requestSocket.setsockopt(zmq.LINGER, 0)
requestSocket.send_multipart([codes.pcaAsksForConfig, id.encode()])
try:
#returns None in pca is unknown
configJSON = requestSocket.recv()
if configJSON == codes.idUnknown:
print("id %s is not in Database" % self.id)
sys.exit(1)
configJSON = json.loads(configJSON.decode())
configECS = partitionDataObject(configJSON)
except zmq.Again:
print("timeout getting configuration")
continue
except zmq.error.ContextTerminated:
pass
finally:
requestSocket.close()
#init stuff
self.detectors = MapWrapper()
#update number
self.sequence = 0
self.transitionNumber = 0
#the only one who may change the status Map, is the publisher thread
self.statusMap = MapWrapper()
self.sem = threading.Semaphore()
self.publishQueue = Queue()
self.autoConfigure = False
self.globalTag = False
self.partitionLocked = False
#ZMQ Socket to publish new state Updates
self.socketPublish = self.context.socket(zmq.PUB)
self.socketPublish.bind("tcp://*:%s" % configECS.portPublish)
#publish logmessages
self.socketLogPublish = self.context.socket(zmq.PUB)
self.socketLogPublish.bind("tcp://*:%s" % configECS.portLog)
#Socket to wait for Updates From Detectors
self.socketDetectorUpdates = self.context.socket(zmq.REP)
self.socketDetectorUpdates.bind("tcp://*:%s" % configECS.portUpdates)
#Socket to serve current statusMap
self.socketServeCurrentStatus = self.context.socket(zmq.ROUTER)
self.socketServeCurrentStatus.bind("tcp://*:%s" %
configECS.portCurrentState)
#socket for receiving commands
self.remoteCommandSocket = self.context.socket(zmq.REP)
self.remoteCommandSocket.bind("tcp://*:%s" % configECS.portCommand)
#init logger
self.logfile = conf["logPath"]
debugMode = bool(conf["debugMode"])
logging.basicConfig(
format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S %p',
#level = logging.DEBUG,
handlers=[
#logging to file
logging.FileHandler(self.logfile),
#logging on console and WebUI
logging.StreamHandler()
])
logging.getLogger().setLevel(logging.INFO)
logging.getLogger().handlers[0].setLevel(logging.INFO)
#set console log to info level if in debug mode
if debugMode:
logging.getLogger().handlers[1].setLevel(logging.INFO)
else:
logging.getLogger().handlers[1].setLevel(logging.CRITICAL)
#get your Detectorlist
detList = None
while detList == None:
requestSocket = self.context.socket(zmq.REQ)
requestSocket.connect("tcp://%s:%s" %
(conf['ECAAddress'], conf['ECARequestPort']))
requestSocket.setsockopt(zmq.RCVTIMEO,
int(conf['receive_timeout']))
requestSocket.setsockopt(zmq.LINGER, 0)
requestSocket.send_multipart(
[codes.pcaAsksForDetectorList,
self.id.encode()])
try:
#receive detectors as json
ret = requestSocket.recv()
if ret == codes.error:
self.log("received error getting DetectorList", True)
continue
detJSON = json.loads(ret.decode())
#create DataObjectCollection from JSON
detList = DataObjectCollection(detJSON, detectorDataObject)
except zmq.Again:
self.log("timeout getting DetectorList", True)
continue
except zmq.error.ContextTerminated:
pass
finally:
requestSocket.close()
self.stateMachine = Statemachine(conf["stateMachineCSV"],
PCAStates.Idle)
#helper maps to determine global state
self.readyDetectors = {}
self.configuringDetectors = {}
start_new_thread(self.publisher, ())
#Subscribers need some time to subscribe
time.sleep(1)
#tell subscribers to reset their state Table
self.publishQueue.put((self.id, codes.reset))
#create global System objects
systemList = ["TFC", "DCS", "QA", "FLES"]
res = []
self.messageHandlerFunctionForSystem = {
"DCS": self.handleDCSMessage,
}
#get info from ECS
for s in systemList:
requestSocket = self.context.socket(zmq.REQ)
requestSocket.connect("tcp://%s:%s" %
(conf['ECAAddress'], conf['ECARequestPort']))
requestSocket.setsockopt(zmq.RCVTIMEO,
int(conf['receive_timeout']))
requestSocket.setsockopt(zmq.LINGER, 0)
requestSocket.send_multipart(
[codes.GlobalSystemAsksForInfo,
s.encode()])
try:
#receive detectors as json
ret = requestSocket.recv()
if ret == codes.error:
self.log("received error getting GlobalSystem %s" % s,
True)
exit(1)
if ret == codes.idUnknown:
self.log("ECS does not know GlobalSystem %s" % s, True)
exit(1)
JSON = json.loads(ret.decode())
#create DataObjectCollection from JSON
res.append(globalSystemDataObject(JSON))
except zmq.Again:
self.log("timeout getting GlobalSystem %s" % s, True)
exit(1)
except zmq.error.ContextTerminated:
pass
finally:
requestSocket.close()
#create Objects
tfcData, dcsData, qaData, flesData = res
self.globalSystems = {}
self.TFC = TFC(self.id, tfcData.address, tfcData.portCommand, "TFC",
self.log, self.globalSystemTimeout,
self.globalSystemReconnect)
self.globalSystems["TFC"] = self.TFC
self.DCS = DCS(self.id, dcsData.address, dcsData.portCommand, "DCS",
self.log, self.globalSystemTimeout,
self.globalSystemReconnect)
self.globalSystems["DCS"] = self.DCS
self.QA = QA(self.id, qaData.address, qaData.portCommand, "QA",
self.log, self.globalSystemTimeout,
self.globalSystemReconnect)
self.globalSystems["QA"] = self.QA
self.FLES = FLES(self.id, flesData.address, flesData.portCommand,
"FLES", self.log, self.globalSystemTimeout,
self.globalSystemReconnect)
self.globalSystems["FLES"] = self.FLES
#maps configure Functions to their corresponding PCA State
self.configureFunctionForState = {
PCAStates.Idle: self.TFC.getReady,
PCAStates.TFC_Active: self.makeDetectorsReady,
PCAStates.Detectors_Active: self.configureDCSandFLES,
PCAStates.FLES_and_DCS_Active: self.QA.getReady,
}
self.configureFunctionForSystem = {
"TFC": self.TFC.getReady,
"Detectors": self.makeDetectorsReady,
"DCS": self.configureDCSandFLES,
"FLES": self.configureDCSandFLES,
"QA": self.QA.getReady,
}
#create Detector objects
threadArray = []
for d in detList:
#in case there of a connection problem, creating a Detector might take a long time, therefore create a own thread for each detector
t = threading.Thread(name='dcreate' + str(d.id),
target=self.addDetector,
args=(d, ))
threadArray.append(t)
t.start()
for t in threadArray:
t.join()
self.publishQueue.put(
(self.id,
stateObject([
self.stateMachine.currentState,
self.stateMachine.currentState, self.globalTag, None
])))
#thread stuff
start_new_thread(self.waitForUpdates, ())
start_new_thread(self.waitForStateTableRequests, ())
start_new_thread(self.waitForCommands, ())
self.initdone.set()