def __init__(self):
# Define names of lock and last_used_setup files
self.lock_file = "run/lock"
self.lus_file = "setup/last_used_setup"
# Redefine print to send output to log file
sys.stdout = Logger()
# Create lock file
if (self.create_lock_file() == "error"): exit(1)
# Define what setup to use at startup (can be changed interactively)
initial_setup = self.get_initial_setup()
print "=== Starting PADME Run Control server with %s setup"%initial_setup
#self.write_log("=== Starting PADME Run Control server with %s setup"%initial_setup)
# Create run
self.run = Run()
if (self.run.change_setup(initial_setup) == "error"):
print "ERROR - Error while changing run setup to %s"%initial_setup
#self.write_log("ERROR - Error while changing run setup to %s"%initial_setup)
exit(1)
# Start in idle state
self.current_state = "idle"
# Create handler for PadmeDB
self.db = PadmeDB()
# Create useful regular expressions
self.re_get_board_config = re.compile("^get_board_config (\d+)$")
self.re_get_board_log_file = re.compile("^get_board_log_file (\d+)$")
self.re_change_setup = re.compile("^change_setup (\w+)$")
# Create a TCP/IP socket
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Bind the socket to the port
server_address = ('localhost', 10000)
print "Starting server socket on %s port %s"%server_address
#self.write_log("Starting server socket on %s port %s"%server_address)
try:
self.sock.bind(server_address) # Listen for incoming connections
except:
print "ERROR - Could not bind to socket: %s"%str(sys.exc_info()[0])
#self.write_log("ERROR - Could not bind to socket: %s"%str(sys.exc_info()[0]))
exit(1)
self.sock.listen(1)
# Define SIGINT handler
signal.signal(signal.SIGINT,self.sigint_handler)
# Setup main interface
self.main_loop()
# Clean up before exiting
self.sock.close()
def create_run(self):
if (self.run_number):
# Connect to DB
db = PadmeDB()
db.create_run(self.run_number,self.run_type,self.run_user,self.run_comment)
# Write run configuration parameters to DB
db.add_cfg_para(self.run_number,"setup", self.setup)
s_board_list = ""
for s_bid in self.s_boards: s_board_list += " "+s_bid
db.add_cfg_para(self.run_number,"board_list", s_board_list)
db.add_cfg_para(self.run_number,"config_dir", self.config_dir)
db.add_cfg_para(self.run_number,"config_file", self.config_file)
db.add_cfg_para(self.run_number,"config_file_head", self.config_file_head)
db.add_cfg_para(self.run_number,"log_dir", self.log_dir)
db.add_cfg_para(self.run_number,"log_file_head", self.log_file_head)
db.add_cfg_para(self.run_number,"start_file", self.start_file)
db.add_cfg_para(self.run_number,"quit_file", self.quit_file)
db.add_cfg_para(self.run_number,"initok_file_head", self.lock_file_head)
db.add_cfg_para(self.run_number,"initfail_file_head", self.lock_file_head)
db.add_cfg_para(self.run_number,"lock_file_head", self.lock_file_head)
#db.add_cfg_para(self.run_number,"db_file", self.db_file)
db.add_cfg_para(self.run_number,"data_dir", self.data_dir)
db.add_cfg_para(self.run_number,"data_file_head", self.data_file_head)
db.add_cfg_para(self.run_number,"total_daq_time", self.total_daq_time)
del db
class RunControlServer:
def __init__(self):
# Define names of lock and last_used_setup files
self.lock_file = "run/lock"
self.lus_file = "setup/last_used_setup"
# Redefine print to send output to log file
sys.stdout = Logger()
# Create lock file
if (self.create_lock_file() == "error"): exit(1)
# Define what setup to use at startup (can be changed interactively)
initial_setup = self.get_initial_setup()
print "=== Starting PADME Run Control server with %s setup"%initial_setup
#self.write_log("=== Starting PADME Run Control server with %s setup"%initial_setup)
# Create run
self.run = Run()
if (self.run.change_setup(initial_setup) == "error"):
print "ERROR - Error while changing run setup to %s"%initial_setup
#self.write_log("ERROR - Error while changing run setup to %s"%initial_setup)
exit(1)
# Start in idle state
self.current_state = "idle"
# Create handler for PadmeDB
self.db = PadmeDB()
# Create useful regular expressions
self.re_get_board_config = re.compile("^get_board_config (\d+)$")
self.re_get_board_log_file = re.compile("^get_board_log_file (\d+)$")
self.re_change_setup = re.compile("^change_setup (\w+)$")
# Create a TCP/IP socket
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Bind the socket to the port
server_address = ('localhost', 10000)
print "Starting server socket on %s port %s"%server_address
#self.write_log("Starting server socket on %s port %s"%server_address)
try:
self.sock.bind(server_address) # Listen for incoming connections
except:
print "ERROR - Could not bind to socket: %s"%str(sys.exc_info()[0])
#self.write_log("ERROR - Could not bind to socket: %s"%str(sys.exc_info()[0]))
exit(1)
self.sock.listen(1)
# Define SIGINT handler
signal.signal(signal.SIGINT,self.sigint_handler)
# Setup main interface
self.main_loop()
# Clean up before exiting
self.sock.close()
def create_lock_file(self):
# Check if lock file exists
if (os.path.exists(self.lock_file)):
if (os.path.isfile(self.lock_file)):
pid = 0
lf = open(self.lock_file,"r")
for ll in lf: pid = ll
lf.close()
print "ERROR - Lock file %s found for pid %s"%(self.lock_file,pid)
#self.write_log("ERROR - Lock file %s found for pid %s"%(self.lock_file,pid))
else:
print "ERROR - Lock file %s found but it is not a file"%self.lock_file
#self.write_log("ERROR - Lock file %s found but it is not a file"%self.lock_file)
return "error"
# Create our own lock file
pid = os.getpid()
lf = open(self.lock_file,"w")
lf.write("%d"%pid)
lf.close()
return "ok"
def get_initial_setup(self):
setup = "test"
lus = ""
if (os.path.exists(self.lus_file)):
if (os.path.isfile(self.lus_file)):
lusf = open(self.lus_file,"r")
lus = lusf.read().strip("\n")
lusf.close()
if (lus == ""):
print "WARNING - File with last used setup %s exists but it is empty - Using default setup %s"%(self.lus_file,setup)
#self.write_log("WARNING - File with last used setup %s exists but it is empty - Using default setup %s"%(self.lus_file,setup))
else:
setup = lus
else:
print "WARNING - File with last used setup %s exists but it is not a file - Using default setup %s"%(self.lus_file,setup)
#self.write_log("WARNING - File with last used setup %s exists but it is not a file - Using default setup %s"%(self.lus_file,setup))
else:
print "WARNING - Could not find file with last used setup %s - Using default setup %s"%(self.lus_file,setup)
#self.write_log("WARNING - Could not find file with last used setup %s - Using default setup %s"%(self.lus_file,setup))
return setup
def sigint_handler(self,signal,frame):
print "RunControlSever received SIGINT: exiting"
# If a run is initialized/running, abort it as cleanly as possible
if ( self.current_state == "initialized" or self.current_state == "running" ):
self.run.run_comment_end = "Run aborted because of SIGINT"
print "Aborting run on SIGINT"
#self.write_log("Aborting run on SIGINT")
if (self.run.run_number):
self.db.set_run_status(self.run.run_number,4) # Status 4: run aborted
self.db.set_run_time_stop(self.run.run_number,self.now_str())
self.db.set_run_comment_end(self.run.run_number,self.run.run_end_comment)
open(self.run.quit_file,'w').close()
for adc in (self.run.adcboard_list):
if adc.stop_daq():
print "ADC board %02d - Terminated correctly"%adc.board_id
#self.write_log("ADC board %02d - Terminated correctly"%adc.board_id)
else:
print "ADC board %02d - WARNING: problems while terminating DAQ"%adc.board_id
#self.write_log("ADC board %02d - WARNING: problems while terminating DAQ"%adc.board_id)
if (self.run.run_number):
self.db.set_run_status(self.run.run_number,6) # Status 6: run ended with errors
# Clean up run directory
for adc in (self.run.adcboard_list):
if (os.path.exists(adc.initok_file)): os.remove(adc.initok_file)
if (os.path.exists(adc.initfail_file)): os.remove(adc.initfail_file)
if(os.path.exists(self.run.start_file)): os.remove(self.run.start_file)
if(os.path.exists(self.run.quit_file)): os.remove(self.run.quit_file)
if os.path.exists(self.lock_file): os.remove(self.lock_file)
# Now we can exit
exit(0)
def main_loop(self):
while True:
# Wait for a connection
print "Waiting for a connection"
#self.write_log('waiting for a connection')
(self.connection,client_address) = self.sock.accept()
print "Connection from %s"%str(client_address)
#self.write_log('connection from '+str(client_address))
while True:
# Handle connection according to curren status of RunControl
if self.current_state == "idle":
new_state = self.state_idle()
elif self.current_state == "initialized":
new_state = self.state_initialized()
elif self.current_state == "running":
new_state = self.state_running()
elif self.current_state == "initfail":
new_state = self.state_initfail()
else:
print "ERROR: unknown state %s - ABORTING"%self.current_state
#self.write_log("ERROR: unknown state %s - ABORTING"%self.current_state)
new_state = "exit"
# See if status changed
if new_state == "idle" or new_state == "initialized" or new_state == "running" or new_state == "initfail":
self.current_state = new_state
elif new_state == "client_close":
self.connection.close()
break
elif new_state == "exit":
print "=== RunControlSever received exit command: exiting"
self.connection.close()
if os.path.exists(self.lock_file): os.remove(self.lock_file)
exit(0)
else:
print "=== RunControlServer = ERROR: unknown new state %s - ABORTING"%new_state
#self.write_log("ERROR: unknown new state %s - ABORTING"%new_state)
self.connection.close()
if os.path.exists(self.lock_file): os.remove(self.lock_file)
exit(1)
def write_log(self,msg):
print self.now_str()+" "+msg
def now_str(self):
return time.strftime("%Y-%m-%d %H:%M:%S",time.localtime())
def state_idle(self):
# Receive and process commands for "idle" state
while True:
cmd = self.get_command()
print "Received command %s"%cmd
#self.write_log('Received command '+cmd)
if (cmd == "client_close"):
return "client_close"
elif (cmd == "get_state"):
self.send_answer(self.current_state)
elif (cmd == "get_setup"):
self.send_answer(self.run.setup)
elif (cmd == "get_setup_list"):
self.send_answer(self.get_setup_list())
elif (cmd == "get_board_list"):
self.send_answer(str(self.run.boardid_list))
elif (cmd == "get_run_number"):
self.send_answer(str(self.db.get_last_run_in_db()))
elif (cmd == "new_run"):
res = self.new_run()
if (res == "client_close"):
return "client_close"
elif (res == "error"):
#self.write_log('ERROR while initializing new run')
print "ERROR while initializing new run"
elif (res == "initialized"):
return "initialized"
elif (res == "initfail"):
return "initfail"
else:
#self.write_log("ERROR: new_run returned unknown answer "+res+" (?)")
print "ERROR: new_run returned unknown answer %s (?)"%res
elif (cmd == "exit"):
self.send_answer("exiting")
return "exit"
elif (cmd == "help"):
msg = """Available commands:
help\t\tShow this help
get_state\tShow current state of RunControl
get_setup\tShow current setup name
get_setup_list\tShow list of available setups
get_board_list\tShow list of boards in use with current setup
get_board_config <b>\tShow current configuration of board <b>
get_run_number\tReturn last run number in DB
change_setup <setup>\tChange run setup to <setup>
new_run\t\tInitialize system for a new run
exit\t\tTell RunControl server to exit (use with extreme care!)"""
self.send_answer(msg)
else:
# See if command can be handled by a regular expression
found_re = False
m = self.re_get_board_config.match(cmd)
if (m):
self.send_answer(self.get_board_config(int(m.group(1))))
found_re = True
m = self.re_change_setup.match(cmd)
if (m):
self.send_answer(self.change_setup(m.group(1)))
found_re = True
# No regular expression matched: command is unknown
if not found_re:
self.send_answer("unknown command")
#self.write_log('command '+cmd+' is unknown')
print "Command %s is unknown"%cmd
def state_initialized(self):
# Receive and process commands for "initialized" state
while True:
cmd = self.get_command()
#self.write_log('received command '+cmd)
print "Received command %s"%cmd
if (cmd == "client_close"):
return "client_close"
elif (cmd == "get_state"):
self.send_answer(self.current_state)
elif (cmd == "get_setup"):
self.send_answer(self.run.setup)
elif (cmd == "get_board_list"):
self.send_answer(str(self.run.boardid_list))
elif (cmd == "get_run_number"):
self.send_answer(str(self.run.run_number))
elif (cmd == "abort_run"):
return self.abort_run()
elif (cmd == "start_run"):
return self.start_run()
elif (cmd == "exit"):
self.send_answer("exiting")
return "exit"
elif (cmd == "help"):
msg = """Available commands:
help\t\tShow this help
get_state\tShow current state of RunControl
get_setup\tShow current setup name
get_board_list\tShow list of boards in use with current setup
get_board_config <b>\tShow current configuration of board <b>
get_board_log_file <b>\tGet name of log file for board <b>
get_run_number\tReturn current run number
start_run\t\tStart run
abort_run\t\tAbort run
exit\t\tTell RunControl server to exit (use with extreme care!)"""
self.send_answer(msg)
else:
# See if command can be handled by a regular expression
found_re = False
m = self.re_get_board_config.match(cmd)
if (m):
self.send_answer(self.get_board_config(int(m.group(1))))
found_re = True
m = self.re_get_board_log_file.match(cmd)
if (m):
self.send_answer(self.get_board_log_file(int(m.group(1))))
found_re = True
# No regular expression matched: command is unknown
if not found_re:
self.send_answer("unknown command")
#self.write_log('command '+cmd+' is unknown')
print "Command %s is unknown"%cmd
def state_running(self):
# Receive and process commands for "running" state
while True:
cmd = self.get_command()
#self.write_log('received command '+cmd)
print "Received command %s"%cmd
if (cmd == "client_close"):
return "client_close"
elif (cmd == "get_state"):
self.send_answer(self.current_state)
elif (cmd == "get_setup"):
self.send_answer(self.run.setup)
elif (cmd == "get_board_list"):
self.send_answer(str(self.run.boardid_list))
elif (cmd == "get_run_number"):
self.send_answer(str(self.run.run_number))
elif (cmd == "stop_run"):
return self.stop_run()
elif (cmd == "exit"):
self.send_answer("exiting")
return "exit"
elif (cmd == "help"):
msg = """Available commands:
help\t\tShow this help
get_state\tShow current state of RunControl
get_setup\tShow current setup name
get_board_list\tShow list of boards in use with current setup
get_board_config <b>\tShow current configuration of board <b>
get_board_log_file <b>\tGet name of log file for board <b>
get_run_number\tReturn current run number
stop_run\t\tStop the run
exit\t\tTell RunControl server to exit (use with extreme care!)"""
self.send_answer(msg)
else:
# See if command can be handled by a regular expression
found_re = False
m = self.re_get_board_config.match(cmd)
if (m):
self.send_answer(self.get_board_config(int(m.group(1))))
found_re = True
m = self.re_get_board_log_file.match(cmd)
if (m):
self.send_answer(self.get_board_log_file(int(m.group(1))))
found_re = True
# No regular expression matched: command is unknown
if not found_re:
self.send_answer("unknown command")
#self.write_log('command '+cmd+' is unknown')
print "Command %s is unknown"%cmd
return "idle"
def state_initfail(self):
return "idle"
def get_command(self):
# First get length of string
l = ""
for i in range(5): # Max 99999 characters
ch = self.connection.recv(1)
if ch:
l += ch
else:
#self.write_log('no more data from client')
print "Client closed connection"
return "client_close"
ll = int(l)
# Then read the right amount of characters from the socket
cmd = ""
for i in range(ll):
ch = self.connection.recv(1)
if ch:
cmd += ch
else:
#self.write_log('no more data from client')
print "Client closed connection"
return "client_close"
return cmd
def send_answer(self,answer):
if len(answer)<100000:
#self.write_log("Sending answer "+answer)
print "Sending answer %s"%answer
self.connection.sendall("%5.5d"%len(answer)+answer)
else:
#self.write_log('answer too long: cannot send')
print "Answer is too long: cannot send"
def get_board_config(self,brdid):
if brdid in self.run.boardid_list:
return self.run.adcboard_list[self.run.boardid_list.index(brdid)].format_config()
else:
return "ERROR: board id %d does not exist"%brdid
def get_board_log_file(self,brdid):
if brdid in self.run.boardid_list:
return self.run.adcboard_list[self.run.boardid_list.index(brdid)].log_file
else:
return "ERROR: board id %d does not exist"%brdid
def read_setup_list(self):
# Get list of available setups
setup_main_dir = "setup"
setups = []
for top,dirs,files in os.walk(setup_main_dir):
if (top == setup_main_dir):
for setup_dir in dirs: setups.append(setup_dir)
setups.sort()
return setups
def get_setup_list(self):
return str(self.read_setup_list())
def change_setup(self,setup):
# Check if requested setup is known
if not (setup in self.read_setup_list()):
#self.write_log("change_setup - ERROR: request to set unknown setup "+setup)
print "change_setup - ERROR: request to set unknown setup %s"%setup
return "error"
# Change (or reload) setup
if (setup==self.run.setup):
#self.write_log("change_setup - reloading setup "+setup)
print "change_setup - reloading setup %s"%setup
else:
#self.write_log("change_setup - changing setup from "+self.run.setup+" to "+setup)
print "change_setup - changing setup from %s to %s"%(self.run.setup,setup)
self.run.change_setup(setup)
return setup
def new_run(self):
# Retrieve run number - next=next run from DB, dummy=dummy run (i.e. run nr=0)
# Return run number used (0 for dummy run) or "error" for invalid answer
newrun_number = 0
self.send_answer("run_number")
ans = self.get_command()
if (ans=="next"):
newrun_number = self.db.get_last_run_in_db()+1
elif (ans=="dummy"):
newrun_number = 0
elif (ans == "error"):
#self.write_log("run_number - client returned error")
print "run_number - client returned error"
return "error"
elif (ans=="client_close"):
return "client_close"
else:
#self.write_log("run_number - invalid option %s received"%ans)
print "run_number - invalid option %s received"%ans
self.send_answer("error")
return "error"
self.send_answer(str(newrun_number))
# Retrieve run type (TEST,DAQ,COSMIC)
# Return run type used or "error" for invalid answer
newrun_type = ""
self.send_answer("run_type")
ans = self.get_command()
if (ans=="TEST" or ans=="DAQ" or ans=="COSMIC"):
newrun_type = ans
elif (ans == "error"):
self.write_log("run_type - client returned error")
return "error"
elif (ans=="client_close"):
return "client_close"
else:
# self.write_log("run_type - invalid option %s received"%ans)
print "run_type - invalid option %s received"%ans
self.send_answer("error")
return "error"
self.send_answer(newrun_type)
newrun_user = ""
self.send_answer("shift_crew")
ans = self.get_command()
if (ans=="client_close"): return "client_close"
newrun_user = ans
newrun_comment = ""
self.send_answer("run_comment")
ans = self.get_command()
if (ans=="client_close"): return "client_close"
newrun_comment = ans
#self.write_log("Run number: "+str(newrun_number))
#self.write_log("Run type: "+newrun_type)
#self.write_log("Run crew: "+newrun_user)
#self.write_log("Run comment: "+newrun_comment)
print "Run number: %d"%newrun_number
print "Run type: %s"%newrun_type
print "Run crew: %s"%newrun_user
print "Run comment: %s"%newrun_comment
# Set run configuration according to user's request
self.run.change_run(newrun_number)
self.run.run_type = newrun_type
self.run.run_user = newrun_user
self.run.run_comment = newrun_comment
# Check if requested run number was not used before
# Saves the day if more than one RunControl program is running at the same time (DON'T DO THAT!!!)
if (self.run.run_number):
run_is_in_db = self.db.is_run_in_db(self.run.run_number)
if (run_is_in_db):
#self.write_log("ERROR - Run "+str(self.run.run_number)+" is already in the DB: cannot use it again")
#self.write_log("Please check if someone else is using this RunControl before retrying")
print "ERROR - Run %d is already in the DB: cannot use it again"%self.run.run_number
print "Please check if someone else is using this RunControl before retrying"
self.send_answer("error_init")
return "error"
# Create run structure in the DB
#self.write_log("Initializing Run "+str(self.run.run_number))
print "Initializing Run %d"%self.run.run_number
self.run.create_run()
if (self.run.run_number): self.db.set_run_time_init(self.run.run_number,self.now_str())
# Create directory to host log files
#self.write_log("Creating log directory "+self.run.log_dir)
print "Creating log directory %s"%self.run.log_dir
self.run.create_log_dir()
# Write run and boards configuration files
#self.write_log("Writing configuration file "+self.run.config_file)
print "Writing configuration file %s"%self.run.config_file
self.run.write_config()
for adc in (self.run.adcboard_list):
#self.write_log("Writing configuration file "+adc.config_file)
print "Writing configuration file %s"%adc.config_file
adc.write_config()
# Start DAQ for all boards
self.send_answer("start_init")
for adc in (self.run.adcboard_list):
p_id = adc.start_daq()
if p_id:
#self.write_log("ADC board %02d - Started DAQ with process id %d"%(adc.board_id,p_id))
print "ADC board %02d - Started DAQ with process id %d"%(adc.board_id,p_id)
self.send_answer("adc "+str(adc.board_id)+" init")
adc.status = "init"
else:
#self.write_log("ADC board %02d - ERROR: could not start DAQ"%adc.board_id)
print "ADC board %02d - ERROR: could not start DAQ"%adc.board_id
self.send_answer("adc "+str(adc.board_id)+" fail")
adc.status = "fail"
# Wait for all boards to finish initialization
n_try = 0
while(1):
all_boards_init = 1
all_boards_ready = 1
for adc in (self.run.adcboard_list):
# Check if any board changed status
if (adc.status == "init"):
if (os.path.exists(adc.initok_file)):
# Initialization ended OK
#self.write_log("ADC board %02d - Initialized and ready for DAQ"%adc.board_id)
print "ADC board %02d - Initialized and ready for DAQ"%adc.board_id
self.send_answer("adc "+str(adc.board_id)+" ready")
adc.status = "ready"
elif (os.path.exists(adc.initfail_file)):
# Problem during initialization
#self.write_log("ADC board %02d - *** Initialization failed ***"%adc.board_id)
print "ADC board %02d - *** Initialization failed ***"%adc.board_id
self.send_answer("adc "+str(adc.board_id)+" fail")
adc.status = "fail"
else:
# This board is still initializing
all_boards_init = 0
# Check if any board is in fail status
if (adc.status == "fail"): all_boards_ready = 0
if (all_boards_init == 0):
# Some boards are still initializing: keep waiting
n_try += 1
if (n_try>=10):
#self.write_log("*** ERROR *** One or more boards did not initialize within 10sec. Cannot start run")
print "*** ERROR *** One or more boards did not initialize within 10sec. Cannot start run"
if (self.run.run_number): self.db.set_run_status(self.run.run_number,5) # Status 5: run with problems at initialization
self.send_answer("init_timeout")
return "error"
time.sleep(1)
elif (all_boards_ready):
#self.write_log("All boards completed initialization: DAQ run can be started")
print "All boards completed initialization: DAQ run can be started"
if (self.run.run_number): self.db.set_run_status(self.run.run_number,1) # Status 1: run correctly initialized
self.send_answer("init_ready")
return "initialized"
else:
#self.write_log("*** ERROR *** One or more boards failed the initialization. Cannot start run")
print "*** ERROR *** One or more boards failed the initialization. Cannot start run"
if (self.run.run_number): self.db.set_run_status(self.run.run_number,5) # Status 5: run with problems at initialization
self.send_answer("init_fail")
return "initfail"
def start_run(self):
#self.write_log("Starting run")
print "Starting run"
if (self.run.run_number):
self.db.set_run_time_start(self.run.run_number,self.now_str())
self.db.set_run_status(self.run.run_number,2) # Status 2: run started
# Create "start the run" tag file
open(self.run.start_file,'w').close()
self.send_answer("run_started")
# RunControl is now in "running" mode
return "running"
def stop_run(self):
self.send_answer("run_comment_end")
ans = self.get_command()
if (ans=="client_close"): return "client_close"
#self.write_log("End of Run comment: "+ans)
print "End of Run comment: %s"%ans
self.run.run_comment_end = ans
#self.write_log("Stopping run")
print "Stopping run"
if (self.run.run_number): self.db.set_run_status(self.run.run_number,3) # Status 3: run stopped normally
return self.terminate_run()
def abort_run(self):
self.run.run_comment_end = "Run aborted"
#self.write_log("Aborting run")
print "Aborting run"
if (self.run.run_number): self.db.set_run_status(self.run.run_number,4) # Status 4: run aborted
return self.terminate_run()
def terminate_run(self):
if (self.run.run_number):
self.db.set_run_time_stop(self.run.run_number,self.now_str())
self.db.set_run_comment_end(self.run.run_number,self.run.run_end_comment)
# Create "stop the run" tag file
open(self.run.quit_file,'w').close()
# Run stop_daq procedure for each ADC board
terminate_ok = True
for adc in (self.run.adcboard_list):
if adc.stop_daq():
self.send_answer("adc %d terminate_ok"%adc.board_id)
#self.write_log("ADC board %02d - Terminated correctly"%adc.board_id)
print "ADC board %02d - Terminated correctly"%adc.board_id
else:
terminate_ok = False
self.send_answer("adc %d terminate_error"%adc.board_id)
#self.write_log("ADC board %02d - WARNING: problems while terminating DAQ"%adc.board_id)
print "ADC board %02d - WARNING: problems while terminating DAQ"%adc.board_id
if (self.run.run_number): self.db.set_run_status(self.run.run_number,6) # Status 6: run ended with errors
# Clean up run directory
for adc in (self.run.adcboard_list):
if (os.path.exists(adc.initok_file)): os.remove(adc.initok_file)
if (os.path.exists(adc.initfail_file)): os.remove(adc.initfail_file)
if(os.path.exists(self.run.start_file)): os.remove(self.run.start_file)
if(os.path.exists(self.run.quit_file)): os.remove(self.run.quit_file)
if terminate_ok:
self.send_answer("terminate_ok")
else:
self.send_answer("terminate_error")
# At the end of this procedure RunControl is back to "idle" mode
return "idle"
def now_str(self): return time.strftime("%Y-%m-%d %H:%M:%S",time.localtime())
class ADCBoard:
def __init__(self,b_id):
self.board_id = b_id
# Get position of DAQ main directory from PADME_DAQ_DIR environment variable
# Default to current dir if not set
self.daq_dir = os.getenv('PADME_DAQ_DIR',".")
# Define id file for passwordless ssh command execution
self.ssh_id_file = "%s/.ssh/id_rsa_daq"%os.getenv('HOME',"~")
self.db = PadmeDB()
self.status = "idle"
self.re_dbinfo_line = re.compile("^\s*DBINFO\s.*$")
self.set_default_config()
def set_default_config(self):
self.node_id = 0
self.node_ip = ""
self.conet2_link = -1
self.conet2_slot = -1
self.executable = os.getenv('PADME',".")+"/PadmeDAQ/PadmeDAQ.exe"
self.run_number = 0
self.proc_daq_id = -1
self.proc_zsup_id = -1
self.process_mode = "DAQ"
self.config_file_daq = "unset"
self.log_file_daq = "unset"
self.lock_file_daq = "unset"
self.initok_file_daq = "unset"
self.initfail_file_daq = "unset"
self.output_mode_daq = "STREAM"
self.output_stream_daq = "unset"
self.data_dir_daq = "unset"
self.data_file_daq = "daq"
self.config_file_zsup = "unset"
self.log_file_zsup = "unset"
self.lock_file_zsup = "unset"
self.initok_file_zsup = "unset"
self.initfail_file_zsup = "unset"
self.input_stream_zsup = "unset"
self.output_mode_zsup = "STREAM"
self.output_stream_zsup = "unset"
self.data_dir_zsup = "unset"
self.data_file_zsup = "zsup"
self.start_file = "unset"
self.quit_file = "unset"
self.total_daq_time = 0
# Default V1742 ADC settings
self.startdaq_mode = 0
self.trigger_mode = 1
self.trigger_iolevel = "NIM"
self.group_enable_mask = int('0xf',0)
self.channel_enable_mask = int('0xffffffff',0)
self.offset_global = int('0x5600',0)
self.offset_ch = []
for ch in range(32): self.offset_ch.append(self.offset_global)
self.post_trigger_size = 65
self.max_num_events_blt = 128
self.drs4_sampfreq = 2
self.drs4corr_enable = 1
# Autopass parameters (threshold in counts, duration in ns)
self.auto_threshold = int('0x0400',0)
self.auto_duration = 150
# Default zero suppression settings
self.zero_suppression = 0
self.zs1_head = 80
self.zs1_tail = 30
self.zs1_nsigma = 3.
self.zs1_nabovethr = 4
self.zs1_badrmsthr = 15.
self.zs2_tail = 30
self.zs2_minrms = 4.6
self.zs2_minrms_ch = []
for ch in range(32): self.zs2_minrms_ch.append(self.zs2_minrms)
# Default file parameters
self.file_max_duration = 3600
self.file_max_size = 1024*1024*1024
self.file_max_events = 1000*1000
# Default DAQ control parameters
self.daq_loop_delay = 10000
self.debug_scale = 100
def read_setup(self,setup):
if (self.board_id == -1):
print "ADCBoard - ERROR: board id not set while reading setup"
return
# Define regular expressions used in file parsing
re_empty = re.compile("^\s*$")
re_comment = re.compile("^\s*#")
re_param = re.compile("^\s*(\w+)\s+(.+?)\s*$")
re_chvalue = re.compile("^\s*(\d+)\s+(\w+)\s*$")
# Read default board configuration from file
setup_file = self.daq_dir+"/setup/"+setup+"/board_%02d.cfg"%self.board_id
if (not os.path.isfile(setup_file)):
print "ADCBoard - WARNING: setup file %s not found for board %d"%(setup_file,self.board_id)
return
f = open(setup_file)
for l in f:
if (re_empty.search(l) or re_comment.search(l)): continue
m = re_param.search(l)
if (m):
(p_name,p_value) = m.group(1,2)
if (p_name == "startdaq_mode"): self.startdaq_mode = int(p_value,0)
elif (p_name == "trigger_mode"): self.trigger_mode = int(p_value,0)
elif (p_name == "trigger_iolevel"): self.trigger_iolevel = p_value
elif (p_name == "group_enable_mask"): self.group_enable_mask = int(p_value,0)
elif (p_name == "channel_enable_mask"): self.channel_enable_mask = int(p_value,0)
elif (p_name == "offset_global"):
self.offset_global = int(p_value,0)
for ch in range(0,32): self.offset_ch[ch] = self.offset_global
elif (p_name == "offset_ch"):
mm = re_chvalue.search(p_value)
if (mm):
(ch,offset) = mm.group(1,2)
self.offset_ch[int(ch,0)] = int(offset,0)
else:
print "ADCBoard - ERROR decoding channel offset parameter in line:"
print l
elif (p_name == "post_trigger_size"): self.post_trigger_size = int(p_value,0)
elif (p_name == "max_num_events_blt"): self.max_num_events_blt = int(p_value,0)
elif (p_name == "drs4corr_enable"): self.drs4corr_enable = int(p_value,0)
elif (p_name == "drs4_sampfreq"): self.drs4_sampfreq = int(p_value,0)
elif (p_name == "daq_loop_delay"): self.daq_loop_delay = int(p_value,0)
elif (p_name == "debug_scale"): self.debug_scale = int(p_value,0)
elif (p_name == "file_max_duration"): self.file_max_duration = int(p_value,0)
elif (p_name == "file_max_size"): self.file_max_size = int(p_value,0)
elif (p_name == "file_max_events"): self.file_max_events = int(p_value,0)
elif (p_name == "zero_suppression"): self.zero_suppression = int(p_value,0)
elif (p_name == "zs1_head"): self.zs1_head = int(p_value,0)
elif (p_name == "zs1_tail"): self.zs1_tail = int(p_value,0)
elif (p_name == "zs1_nsigma"): self.zs1_nsigma = float(p_value)
elif (p_name == "zs1_nabovethr"): self.zs1_nabovethr = int(p_value,0)
elif (p_name == "zs2_tail"): self.zs2_tail = int(p_value,0)
elif (p_name == "zs2_minrms"):
self.zs2_minrms = float(p_value)
for ch in range(32): self.zs2_minrms_ch[ch] = self.zs2_minrms
elif (p_name == "zs2_minrms_ch"):
mm = re_chvalue.search(p_value)
if (mm):
(ch,minrms) = mm.group(1,2)
self.zs2_minrms_ch[int(ch,0)] = int(minrms,0)
else:
print "ADCBoard - ERROR decoding channel zs2_minrms parameter in line:"
print l
elif (p_name == "auto_threshold"): self.auto_threshold = int(p_value,0)
elif (p_name == "auto_duration"): self.auto_duration = int(p_value,0)
else:
print "ADCBoard - WARNNING: unknown parameter found while reading default config file: %s"%p_name
else:
print "ADCBoard WARNING: unknown line format found while reading default config file"
print l
f.close()
def config_list_daq(self):
cfg_list = []
cfg_list.append(["daq_dir", self.daq_dir])
cfg_list.append(["ssh_id_file", self.ssh_id_file])
cfg_list.append(["executable", self.executable])
cfg_list.append(["start_file", self.start_file])
cfg_list.append(["quit_file", self.quit_file])
cfg_list.append(["run_number", str(self.run_number)])
cfg_list.append(["board_id", str(self.board_id)])
cfg_list.append(["process_mode", self.process_mode])
#if (self.run_number):
# cfg_list.append(["process_id", str(self.proc_daq_id)])
cfg_list.append(["node_id", str(self.node_id)])
cfg_list.append(["node_ip", self.node_ip])
cfg_list.append(["conet2_link", str(self.conet2_link)])
cfg_list.append(["conet2_slot", str(self.conet2_slot)])
cfg_list.append(["config_file", self.config_file_daq])
cfg_list.append(["log_file", self.log_file_daq])
cfg_list.append(["lock_file", self.lock_file_daq])
cfg_list.append(["initok_file", self.initok_file_daq])
cfg_list.append(["initfail_file", self.initfail_file_daq])
cfg_list.append(["output_mode", self.output_mode_daq])
if self.output_mode_daq == "STREAM":
cfg_list.append(["output_stream", self.output_stream_daq])
elif self.output_mode_daq == "FILE":
cfg_list.append(["data_dir", self.data_dir_daq])
cfg_list.append(["data_file", self.data_file_daq])
cfg_list.append(["file_max_duration",str(self.file_max_duration)])
cfg_list.append(["file_max_size", str(self.file_max_size)])
cfg_list.append(["file_max_events", str(self.file_max_events)])
cfg_list.append(["total_daq_time", str(self.total_daq_time)])
cfg_list.append(["startdaq_mode", str(self.startdaq_mode)])
cfg_list.append(["trigger_mode", str(self.trigger_mode)])
cfg_list.append(["trigger_iolevel", self.trigger_iolevel])
cfg_list.append(["group_enable_mask", "%#1.1x"%self.group_enable_mask])
cfg_list.append(["channel_enable_mask", "%#8.8x"%self.channel_enable_mask])
cfg_list.append(["offset_global", "%#4.4x"%self.offset_global])
for ch in range(32):
if (self.offset_ch[ch] != self.offset_global):
cfg_list.append(["offset_ch", "%2d %#4.4x"%(ch,self.offset_ch[ch])])
cfg_list.append(["post_trigger_size", str(self.post_trigger_size)])
cfg_list.append(["max_num_events_blt", str(self.max_num_events_blt)])
cfg_list.append(["drs4corr_enable", str(self.drs4corr_enable)])
cfg_list.append(["drs4_sampfreq", str(self.drs4_sampfreq)])
cfg_list.append(["auto_threshold", "%#4.4x"%self.auto_threshold])
cfg_list.append(["auto_duration", str(self.auto_duration)])
cfg_list.append(["daq_loop_delay", str(self.daq_loop_delay)])
cfg_list.append(["debug_scale", str(self.debug_scale)])
return cfg_list
def config_list_zsup(self):
cfg_list = []
cfg_list.append(["daq_dir", self.daq_dir])
cfg_list.append(["ssh_id_file", self.ssh_id_file])
cfg_list.append(["executable", self.executable])
cfg_list.append(["run_number", str(self.run_number)])
cfg_list.append(["board_id", str(self.board_id)])
cfg_list.append(["process_mode", "ZSUP"])
#if (self.run_number):
# cfg_list.append(["process_id", str(self.proc_zsup_id)])
cfg_list.append(["node_id", str(self.node_id)])
cfg_list.append(["node_ip", self.node_ip])
cfg_list.append(["config_file", self.config_file_zsup])
cfg_list.append(["log_file", self.log_file_zsup])
cfg_list.append(["lock_file", self.lock_file_zsup])
cfg_list.append(["initok_file", self.initok_file_zsup])
cfg_list.append(["initfail_file", self.initfail_file_zsup])
cfg_list.append(["output_mode", self.output_mode_zsup])
if self.output_mode_zsup == "STREAM":
cfg_list.append(["output_stream", self.output_stream_zsup])
elif self.output_mode_zsup == "FILE":
cfg_list.append(["data_dir", self.data_dir_zsup])
cfg_list.append(["data_file", self.data_file_zsup])
cfg_list.append(["file_max_duration", str(self.file_max_duration)])
cfg_list.append(["file_max_size", str(self.file_max_size)])
cfg_list.append(["file_max_events", str(self.file_max_events)])
cfg_list.append(["input_stream", self.input_stream_zsup])
cfg_list.append(["zero_suppression", str(self.zero_suppression)])
if (self.zero_suppression%100 == 1):
cfg_list.append(["zs1_head", str(self.zs1_head)])
cfg_list.append(["zs1_tail", str(self.zs1_tail)])
cfg_list.append(["zs1_nsigma", str(self.zs1_nsigma)])
cfg_list.append(["zs1_nabovethr", str(self.zs1_nabovethr)])
cfg_list.append(["zs1_badrmsthr", str(self.zs1_badrmsthr)])
elif (self.zero_suppression%100 == 2):
cfg_list.append(["zs2_tail", str(self.zs2_tail)])
cfg_list.append(["zs2_minrms", str(self.zs2_minrms)])
for ch in range(32):
if (self.zs2_minrms_ch[ch] != self.zs2_minrms):
cfg_list.append(["zs2_minrms_ch","%2d %f"%(ch,self.zs2_minrms_ch[ch])])
cfg_list.append(["debug_scale", str(self.debug_scale)])
return cfg_list
def format_config_daq(self):
cfgstring = ""
for cfg in self.config_list_daq(): cfgstring += "%-30s %s\n"%(cfg[0],cfg[1])
return cfgstring
def format_config_zsup(self):
cfgstring = ""
for cfg in self.config_list_zsup(): cfgstring += "%-30s %s\n"%(cfg[0],cfg[1])
return cfgstring
def write_config(self):
if self.config_file_daq == "unset":
print "ADCBoard - ERROR: write_config() called but config_file_daq not set!"
return
f = open(self.config_file_daq,"w")
f.write(self.format_config_daq())
f.close()
if self.config_file_zsup == "unset":
print "ADCBoard - ERROR: write_config() called but config_file_zsup not set!"
return
f = open(self.config_file_zsup,"w")
f.write(self.format_config_zsup())
f.close()
def print_config(self):
print self.format_config_daq()
print self.format_config_zsup()
def create_proc_daq(self):
# Create DAQ process in DB
self.proc_daq_id = self.db.create_daq_process(self.run_number,self.node_id)
if self.proc_daq_id == -1:
print "ADCBoard::create_proc_daq - ERROR: unable to create new DAQ process in DB"
return "error"
# Add info about optical link
self.db.add_daq_process_optical_link(self.proc_daq_id,self.node_id,self.conet2_link,self.conet2_slot)
# Add all configuration parameters
for cfg in self.config_list_daq():
self.db.add_cfg_para_proc(self.proc_daq_id,cfg[0],cfg[1])
return "ok"
def create_proc_zsup(self):
# Create ZSUP process in DB
self.proc_zsup_id = self.db.create_zsup_process(self.run_number,self.node_id)
if self.proc_zsup_id == -1:
print "ADCBoard::create_proc_zsup - ERROR: unable to create new ZSUP proces in DB"
return "error"
# Add all configuration parameters
for cfg in self.config_list_zsup():
self.db.add_cfg_para_proc(self.proc_zsup_id,cfg[0],cfg[1])
return "ok"
#def get_link_id(self):
#
# # Convert PadmeDAQ link description to link id from DB
# if (self.node_id == -1 or self.conet2_link == -1 or self.conet2_slot == -1): return -1
# return self.db.get_link_id(self.node_id,self.conet2_link/8,self.conet2_link%8,self.conet2_slot)
def start_daq(self):
command = "%s -c %s"%(self.executable,self.config_file_daq)
# If DAQ process runs on a remote node then start it using passwordless ssh connection
if self.node_id != 0:
command = "ssh -i %s %s '( %s )'"%(self.ssh_id_file,self.node_ip,command)
print "- Start DAQ process for board %d"%self.board_id
print command
print " Log written to %s"%self.log_file_daq
# Open log file
self.log_handle_daq = open(self.log_file_daq,"w")
# Start DAQ process
try:
self.process_daq = subprocess.Popen(shlex.split(command),stdout=self.log_handle_daq,stderr=subprocess.STDOUT,bufsize=1)
except OSError as e:
print "ADCBoard::start_daq - ERROR: Execution failed: %s",e
return 0
# Tag start of process in DB
if self.run_number:
self.db.set_process_time_create(self.proc_daq_id,self.db.now_str())
# Return process id
return self.process_daq.pid
def stop_daq(self):
# Wait up to 5 seconds for DAQ to stop of its own (on quit file or on time elapsed)
for i in range(10):
if self.process_daq.poll() != None:
# Process exited: clean up defunct process and close log file
self.process_daq.wait()
self.log_handle_daq.close()
if self.run_number:
self.db.set_process_time_end(self.proc_daq_id,self.db.now_str())
return True
time.sleep(0.5)
# Process did not stop: try sending and interrupt
print "ADCBoard::stop_daq- WARNING: DAQ process did not stop on its own. Sending interrupt"
if self.node_id == 0:
# If process is on local host, just send a kill signal
command = "kill %d"%self.process_daq.pid
else:
# If it is on a remote host, use ssh to send kill command.
# PID on remote host is recovered from the lock file
command = "ssh -i %s %s '( kill `cat %s` )'"%(self.ssh_id_file,self.node_ip,self.lock_file_daq)
print command
os.system(command)
# Wait up to 5 seconds for DAQ to stop on interrupt
for i in range(10):
if self.process_daq.poll() != None:
# Process exited: clean up defunct process and close log file
self.process_daq.wait()
self.log_handle_daq.close()
if self.run_number:
self.db.set_process_time_end(self.proc_daq_id,self.db.now_str())
return True
time.sleep(0.5)
# Process did not stop smoothly: terminate it
print "ADCBoard::stop_daq - WARNING: DAQ process did not stop on interrupt. Terminating it"
self.process_daq.terminate()
time.sleep(1)
if self.process_daq.poll() != None:
self.process_daq.wait()
self.log_handle_daq.close()
if self.run_number: self.db.set_process_time_end(self.proc_daq_id,self.db.now_str())
return False
def start_zsup(self):
command = "%s -c %s"%(self.executable,self.config_file_zsup)
# If ZSUP process runs on a remote node then start it using passwordless ssh connection
if self.node_id != 0:
command = "ssh -i %s %s '( %s )'"%(self.ssh_id_file,self.node_ip,command)
print "- Start ZSUP process for board %d"%self.board_id
print command
print " Log written to %s"%self.log_file_zsup
# Open log file
self.log_handle_zsup = open(self.log_file_zsup,"w")
# Start ZSUP process
try:
self.process_zsup = subprocess.Popen(shlex.split(command),stdout=self.log_handle_zsup,stderr=subprocess.STDOUT,bufsize=1)
except OSError as e:
print "ADCBoard::start_zsup - ERROR: Execution failed: %s",e
return 0
# Tag start of process in DB
if self.run_number:
self.db.set_process_time_create(self.proc_zsup_id,self.db.now_str())
# Return process id
return self.process_zsup.pid
def stop_zsup(self):
# Tag stop process in DB
# Wait up to 5 seconds for ZSUP to stop
for i in range(10):
if self.process_zsup.poll() != None:
# Process exited: clean up defunct process and close log file
self.process_zsup.wait()
self.log_handle_zsup.close()
if self.run_number:
self.db.set_process_time_end(self.proc_zsup_id,self.db.now_str())
return True
time.sleep(0.5)
# Process did not stop smoothly: terminate it
print "ADCBoard::stop_zsup - WARNING: ZSUP process did not stop on interrupt. Terminating it"
self.process_zsup.terminate()
time.sleep(1)
if self.process_zsup.poll() != None:
self.process_zsup.wait()
self.log_handle_zsup.close()
if self.run_number:
self.db.set_process_time_end(self.proc_zsup_id,self.db.now_str())
return False
def parse_log_daq(self):
# Look for DBINFO lines in log file and send them to DBINFO line processor
if os.path.exists(self.log_file_daq) and os.path.isfile(self.log_file_daq):
with open(self.log_file_daq,"r") as log:
for line in log:
if self.re_dbinfo_line.match(line):
self.db.manage_dbinfo_entry(self.proc_daq_id,line)
else:
print "ADCBoard::parse_log_daq - WARNING: DAQ log file %s not found"%self.log_file_daq
def parse_log_zsup(self):
# Look for DBINFO lines in log file and send them to DBINFO line processor
if os.path.exists(self.log_file_zsup) and os.path.isfile(self.log_file_zsup):
with open(self.log_file_zsup,"r") as log:
for line in log:
if self.re_dbinfo_line.match(line):
self.db.manage_dbinfo_entry(self.proc_zsup_id,line)
else:
print "ADCBoard::parse_log_zsup - WARNING: ZSUP log file %s not found"%self.log_file_zsup
class Run:
def __init__(self):
# Get account under which the RunControl runs
self.user_account = os.getenv('USER', "daq")
# Get location of DAQ main directory from PADME_DAQ_DIR
# Default to current dir if not set
self.daq_dir = os.getenv('PADME_DAQ_DIR', ".")
# Get base port number for network tunnels from PADME_RC_TUNNEL_BASE_PORT
self.base_port_number = int(
os.getenv('PADME_RC_TUNNEL_BASE_PORT', "31400"))
# Define id file for passwordless ssh command execution
self.ssh_id_file = "%s/.ssh/id_rsa_daq" % os.getenv('HOME', "~")
# Get location of padme-fw software from PADME
# Default to PADME_DAQ_DIR/padme-fw if not set
self.padme_fw = os.getenv('PADME', "%s/padme-fw" % self.daq_dir)
# Define executables to use in production
self.daq_executable = "%s/PadmeDAQ/PadmeDAQ.exe" % self.padme_fw
self.merger_executable = "%s/Level1/PadmeMerger.exe" % self.padme_fw
self.level1_executable = "%s/Level1/PadmeLevel1.exe" % self.padme_fw
self.trigger_executable = "%s/PadmeTrig/PadmeTrig.exe" % self.padme_fw
# Define directory containing setup subdirectories
self.setup_root_dir = "%s/setup" % self.daq_dir
# Define files where the current and last run name will be written
self.current_run_file = "%s/run/current_run" % self.daq_dir
self.last_run_file = "%s/run/last_run" % self.daq_dir
# Define directory containing rawdata directories for each run
self.rawdata_root_dir = self.daq_dir + "/local/rawdata"
# Define name and position of control files
self.control_dir = self.daq_dir + "/local/run"
self.start_file = self.control_dir + "/start"
self.quit_file = self.control_dir + "/quit"
self.trig_start_file = self.control_dir + "/start_trig"
self.trig_stop_file = self.control_dir + "/stop_trig"
self.initok_file_head = self.control_dir + "/initok"
self.initfail_file_head = self.control_dir + "/initfail"
self.lock_file_head = self.control_dir + "/lock"
# Connect to database services
self.db = PadmeDB()
# Do not define a default setup
self.setup = ""
# Run final status defaults to 3 (stopped normally)
self.final_status = 3
self.set_default_config()
def change_run(self):
# Check if requested run number was not used before
# Saves the day if more than one RunControl program is running at the same time (DON'T DO THAT!!!)
if (self.run_number):
run_is_in_db = self.db.is_run_in_db(self.run_number)
if (run_is_in_db):
print "Run::change_run - ERROR - Run %d is already in the DB: cannot use it again" % self.run_number
print "Please check if someone else is using this RunControl before retrying"
#self.send_answer("error_init")
return False
# Define run name using run number and start time
self.run_name = "run_%7.7d_%s" % (
self.run_number, time.strftime("%Y%m%d_%H%M%S", time.gmtime()))
# Write run name to current_run file for monitoring
with open(self.current_run_file, "w") as lf:
lf.write("%s\n" % self.run_name)
self.run_dir = self.daq_dir + "/runs/" + self.run_name
self.config_dir = "%s/cfg" % self.run_dir
self.config_file = "%s.cfg" % self.run_name
self.config_file_head = self.run_name
self.log_dir = "%s/log" % self.run_dir
self.log_file_head = self.run_name
self.stream_dir = "%s/local/streams/%s" % (self.daq_dir, self.run_name)
self.stream_head = self.run_name
self.rawdata_dir = "%s/%s" % (self.rawdata_root_dir, self.run_name)
self.rawdata_head = self.run_name
# Make sure Merger runs on a different node after each run
self.merger_node = self.next_merger_node()
# Configure Merger for this run
self.runconfig_merger(self.merger)
# Configure Trigger for this run
self.runconfig_trigger(self.trigger)
# Configure ADC boards for this run
for adcboard in self.adcboard_list:
#print "--- Configuring for run board %d"%adcboard.board_id
self.runconfig_adcboard(adcboard)
# Configure Level1 processes for this run
for level1 in self.level1_list:
self.runconfig_level1(level1)
# If this is a real run, create it in the DB
if (self.run_number):
print "Creating Run %d structure in DB" % self.run_number
if self.create_run_in_db() == "error":
print "Run::change_run - ERROR - Cannot create Run in the DB"
return False
return True
def next_merger_node(self):
# If merger node is currently not defined, set it to first in list or localhost
if not self.merger_node:
if self.merger_node_list:
return self.merger_node_list[0]
else:
return "localhost"
# If the list of nodes was not defined, keep using always the same node
if not self.merger_node_list: return self.merger_node
# Find current node in node list and use next one
use_next = False
for node in self.merger_node_list:
if use_next: return node
if node == self.merger_node:
use_next = True
# Current node was the last in the list, so we restart with the first node in the list
if use_next: return self.merger_node_list[0]
# Handle misconfigurations (should never happen)
print "Run::next_merger_node - WARNING: current merger host %s is not in the merger node list %s" % (
self.merger_node, self.merger_node_list)
print " Will use first node in list: %s" % self.merger_node_list[
0]
return self.merger_node_list[0]
def set_default_config(self):
# Clean up Run configuration and set all run parameters to default
self.adcboard_list = []
self.trigger = None
self.merger = None
self.level1_list = []
self.run_number = 0
self.run_name = "run_%7.7d_%s" % (
self.run_number, time.strftime("%Y%m%d_%H%M%S", time.gmtime()))
self.run_type = "TEST"
self.run_user = "******"
self.run_comment_start = "Generic start of run"
self.run_comment_end = "Generic end of run"
self.run_dir = "%s/runs/%s" % (self.daq_dir, self.run_name)
self.config_dir = "%s/cfg" % self.run_dir
self.config_file = "%s.cfg" % self.run_name
self.config_file_head = self.run_name
self.log_dir = "%s/log" % self.run_dir
self.log_file_head = self.run_name
self.stream_dir = "%s/local/streams/%s" % (self.daq_dir, self.run_name)
self.stream_head = self.run_name
self.rawdata_dir = "%s/%s" % (self.rawdata_root_dir, self.run_name)
self.rawdata_head = self.run_name
self.trigger_node = "localhost"
self.merger_node = "localhost"
self.merger_node_list = []
self.level1_nproc = 1
self.level1_maxevt = 10000
self.total_daq_time = 0
def read_setup(self):
# Define regular expressions used in file parsing
re_empty = re.compile("^\s*$")
re_comment = re.compile("^\s*#")
re_param = re.compile("^\s*(\w+)\s+(.+?)\s*$")
re_boardid = re.compile("\d+")
re_board_link = re.compile("^\s*(\d+)\s+(\S+)\s+(\d+)\s+(\d+)\s*$")
# Read general run configuration from file
setup_file = "%s/%s/run.cfg" % (self.setup_root_dir, self.setup)
if (not os.path.isfile(setup_file)):
print "Run - ERROR: setup file %s not found" % setup_file
return "error"
f = open(setup_file)
self.boardid_list = []
self.board_link_list = []
for l in f:
if (re_empty.search(l) or re_comment.search(l)): continue
m = re_param.search(l)
if (m):
(p_name, p_value) = m.group(1, 2)
if (p_name == "boardid_list"):
# Get sorted list of board ids while removing duplicates
s_boards = sorted(list(set(re_boardid.findall(p_value))),
key=int)
for s_bid in s_boards:
# Convert to int as findall returns strings
self.boardid_list.append(int(s_bid))
elif (p_name == "board_link"):
mm = re_board_link.search(p_value)
if (mm):
self.board_link_list.append(mm.group(1, 2, 3, 4))
else:
print "Run - WARNING: unable to decode board_link parameter while reading setup file %s" % (
setup_file, )
print l
elif (p_name == "total_daq_time"):
self.total_daq_time = int(p_value)
elif (p_name == "trigger_node"):
self.trigger_node = p_value
#elif (p_name == "trigger_mask"):
# self.trigger_mask = p_value
elif (p_name == "merger_node"):
self.merger_node = p_value
elif (p_name == "merger_node_list"):
self.merger_node_list = p_value.split()
elif (p_name == "level1_nproc"):
self.level1_nproc = int(p_value)
elif (p_name == "level1_maxevt"):
self.level1_maxevt = int(p_value)
else:
print "Run - WARNING: unknown parameter %s found while reading setup file %s" % (
p_name, setup_file)
else:
print "Run - WARNING: unknown line format found while reading setup file %s" % (
p_name, setup_file)
print l
f.close()
return "ok"
def config_list(self):
cfg_list = []
cfg_list.append(["user_account", self.user_account])
cfg_list.append(["daq_dir", self.daq_dir])
cfg_list.append(["base_port_number", self.base_port_number])
cfg_list.append(["ssh_id_file", self.ssh_id_file])
cfg_list.append(["daq_executable", self.daq_executable])
cfg_list.append(["trigger_executable", self.trigger_executable])
cfg_list.append(["merger_executable", self.merger_executable])
cfg_list.append(["level1_executable", self.level1_executable])
cfg_list.append(["start_file", self.start_file])
cfg_list.append(["quit_file", self.quit_file])
cfg_list.append(["trig_start_file", self.trig_start_file])
cfg_list.append(["trig_stop_file", self.trig_stop_file])
cfg_list.append(["initok_file_head", self.initok_file_head])
cfg_list.append(["initfail_file_head", self.initfail_file_head])
cfg_list.append(["lock_file_head", self.lock_file_head])
cfg_list.append(["rawdata_dir", self.rawdata_dir])
cfg_list.append(["run_number", str(self.run_number)])
cfg_list.append(["run_name", self.run_name])
cfg_list.append(["run_dir", self.run_dir])
cfg_list.append(["run_type", self.run_type])
cfg_list.append(["run_user", self.run_user])
cfg_list.append(["run_comment_start", self.run_comment_start])
cfg_list.append(["setup", self.setup])
s_board_list = ""
for b in self.boardid_list:
if (s_board_list):
s_board_list += " %d" % b
else:
s_board_list = "%d" % b
cfg_list.append(["board_list", s_board_list])
for b in self.boardid_list:
for link in self.board_link_list:
(board, host, port, node) = link
if b == int(board):
board_link = "%s %s %s %s" % (board, host, port, node)
cfg_list.append([
"board_link",
"%s %s %s %s" % (board, host, port, node)
])
cfg_list.append(["config_dir", self.config_dir])
cfg_list.append(["config_file", self.config_file])
cfg_list.append(["config_file_head", self.config_file_head])
cfg_list.append(["log_dir", self.log_dir])
cfg_list.append(["log_file_head", self.log_file_head])
cfg_list.append(["stream_dir", self.stream_dir])
cfg_list.append(["stream_head", self.stream_head])
cfg_list.append(["rawdata_dir", self.rawdata_dir])
cfg_list.append(["rawdata_head", self.rawdata_head])
cfg_list.append(["trigger_node", self.trigger_node])
if self.merger_node:
cfg_list.append(["merger_node", self.merger_node])
if self.merger_node_list:
cfg_list.append(
["merger_node_list", " ".join(self.merger_node_list)])
cfg_list.append(["level1_nproc", str(self.level1_nproc)])
cfg_list.append(["level1_maxevt", str(self.level1_maxevt)])
cfg_list.append(["total_daq_time", str(self.total_daq_time)])
return cfg_list
def format_config(self):
cfgstring = ""
for cfg in self.config_list():
cfgstring += "%-30s %s\n" % (cfg[0], cfg[1])
return cfgstring
def create_run_in_db(self):
# Create run in DB
self.db.create_run(self.run_number, self.run_name, self.run_type)
self.db.set_run_time_create(self.run_number, self.db.now_str())
self.db.set_run_user(self.run_number, self.run_user)
self.db.set_run_comment_start(self.run_number, self.db.now_str(),
self.run_comment_start)
# Add all configuration parameters
for cfg in self.config_list():
self.db.add_cfg_para_run(self.run_number, cfg[0], cfg[1])
# Create board structures in DB
for adc in (self.adcboard_list):
if adc.create_proc_daq() == "error":
print "Run::create_run - ERROR - Cannot create DAQ process for board %d in the DB" % adc.board_id
return "error"
if adc.create_proc_zsup() == "error":
print "Run::create_run - ERROR - Cannot create ZSUP process for board %d in the DB" % adc.board_id
return "error"
# Create Trigger structure in DB
if self.trigger.create_trigger() == "error":
print "Run::create_run - ERROR - Cannot create Trigger process in the DB"
return "error"
# Create Merger structure in DB
if self.merger.create_merger() == "error":
print "Run::create_run - ERROR - Cannot create Merger process in the DB"
return "error"
# Create Level1 structures in DB
for lvl1 in (self.level1_list):
if lvl1.create_level1() == "error":
print "Run::create_run - ERROR - Cannot create Level1 process for level1 %d in the DB" % lvl1.level1_id
return "error"
return "ok"
def create_log_dir(self):
# Create log directory for this run (make sure the full tree is there)
if not os.path.exists(self.log_dir): os.makedirs(self.log_dir, 0755)
def write_config(self):
# Create config directory for this run (make sure the full tree is there)
if not os.path.exists(self.config_dir):
os.makedirs(self.config_dir, 0755)
print "Writing configuration file %s" % self.config_file
f = open(self.config_dir + "/" + self.config_file, "w")
f.write(self.format_config())
f.close()
print "Writing configuration file %s for Merger" % self.merger.config_file
self.merger.write_config()
print "Writing configuration file %s for Trigger" % self.trigger.config_file
self.trigger.write_config()
for adc in (self.adcboard_list):
print "Writing configuration files %s and %s for ADC board %d" % (
adc.config_file_daq, adc.config_file_zsup, adc.board_id)
adc.write_config()
for lvl1 in (self.level1_list):
print "Writing configuration files %s for Level1 %d" % (
lvl1.config_file, lvl1.level1_id)
lvl1.write_config()
def print_config(self):
print self.format_config()
def create_fifos(self):
# Create stream directory and fifo file for Trigger
if self.trigger.node_id == 0:
if not os.path.exists(self.stream_dir):
os.makedirs(self.stream_dir, 0755)
os.mkfifo(self.trigger.output_stream)
else:
command = "ssh -i %s %s '( mkdir -p %s ; mkfifo %s )'" % (
self.ssh_id_file, self.trigger.node_ip, self.stream_dir,
self.trigger.output_stream)
print command
os.system(command)
# Create stream directories and fifo files for ADCBoards
for adc in (self.adcboard_list):
if adc.node_id == 0:
if not os.path.exists(self.stream_dir):
os.makedirs(self.stream_dir, 0755)
os.mkfifo(adc.output_stream_daq)
os.mkfifo(adc.output_stream_zsup)
else:
command = "ssh -i %s %s '( mkdir -p %s ; mkfifo %s %s )'" % (
self.ssh_id_file, adc.node_ip, self.stream_dir,
adc.output_stream_daq, adc.output_stream_zsup)
print command
os.system(command)
# Create stream directory on Merger
if self.merger.node_id == 0:
if not os.path.exists(self.stream_dir):
os.makedirs(self.stream_dir, 0755)
else:
command = "ssh -i %s %s '( mkdir -p %s )'" % (
self.ssh_id_file, self.merger.node_ip, self.stream_dir)
print command
os.system(command)
# Create fifo files to connect Merger to Level1 processes
for lvl1 in (self.level1_list):
if lvl1.node_id == 0:
os.mkfifo(lvl1.input_stream)
else:
command = "ssh -i %s %s '( mkfifo %s )'" % (
self.ssh_id_file, lvl1.node_ip, lvl1.input_stream)
print command
os.system(command)
# If Trigger process runs on a node which is different from the Merger
# we must replicate the Trigger FIFO file on the Merger
if self.trigger.node_id != self.merger.node_id:
# Duplicate Trigger FIFO file on Merger node
if self.merger.node_id == 0:
os.mkfifo(self.trigger.output_stream)
else:
command = "ssh -n -i %s %s '( mkfifo %s )'" % (
self.ssh_id_file, self.merger.node_ip,
self.trigger.output_stream)
print command
os.system(command)
# If an ADCBoard process runs on a node which is different from the Merger
# we must replicate the ZSUP FIFO file on the Merger
if self.merger.node_id == 0:
for adc in (self.adcboard_list):
if adc.node_id != self.merger.node_id:
os.mkfifo(adc.output_stream_zsup)
else:
# More efficient if we give a single (long) ssh command
stream_list = ""
for adc in (self.adcboard_list):
if adc.node_id != self.merger.node_id:
stream_list += " %s" % adc.output_stream_zsup
if stream_list:
command = "ssh -n -i %s %s '( mkfifo %s )'" % (
self.ssh_id_file, self.merger.node_ip, stream_list)
print command
os.system(command)
def create_receivers(self):
# Keep track of receiver processes and handles: needed for final celeanup
self.proc_rcv = []
self.hand_rcv = []
# If an ADCBoard process runs on a node which is different from the Merger
# we create the receiving end of a network tunnel
for adc in (self.adcboard_list):
# Check if ADCBoard and Merger run on different nodes
if adc.node_id != self.merger.node_id:
# Define port for network tunnel
port_number = self.base_port_number + adc.board_id
print "Creating receiving end of network tunnel for board %d on port %d" % (
adc.board_id, port_number)
# Define log file and open it
log_file = "%s/%s_nc_%d_recv.log" % (
self.log_dir, self.log_file_head, port_number)
log_handle = open(log_file, "w")
self.hand_rcv.append(log_handle)
# Open receiving end of tunnel on Merger node
command = "nc -l -k -v --recv-only %s %d > %s < /dev/zero" % (
self.merger.node_ip, port_number, adc.output_stream_zsup)
#command = "nc --udp -l -v --recv-only %s %d > %s < /dev/zero"%(self.merger.node_ip,port_number,adc.output_stream_zsup)
if self.merger.node_id != 0:
command = "ssh -f -i %s %s '( %s )'" % (
self.ssh_id_file, self.merger.node_ip, command)
print command
try:
proc = subprocess.Popen(shlex.split(command),
stdout=log_handle,
stderr=subprocess.STDOUT,
bufsize=1)
self.proc_rcv.append(proc)
except OSError as e:
print "Run::create_receivers - ERROR: Execution failed: %s", e
time.sleep(0.5)
# If the Trigger process runs on a node which is different from the Merger
# we create the receiving end of a network tunnel
if self.trigger.node_id != self.merger.node_id:
# Define port for network tunnel
port_number = self.base_port_number + 99
print "Creating receiving end of network tunnel for trigger on port %d" % port_number
# Define log file and open it
log_file = "%s/%s_nc_%d_recv.log" % (
self.log_dir, self.log_file_head, port_number)
log_handle = open(log_file, "w")
self.hand_rcv.append(log_handle)
# Open receiving end of tunnel on Merger node
#command = "nc -l -k -v --recv-only %s %d > %s < /dev/zero"%(self.merger.node_ip,port_number,self.trigger.output_stream)
command = "nc --udp -l -v --recv-only %s %d > %s < /dev/zero" % (
self.merger.node_ip, port_number, self.trigger.output_stream)
if self.merger.node_id != 0:
command = "ssh -f -i %s %s '( %s )'" % (
self.ssh_id_file, self.merger.node_ip, command)
print command
try:
proc = subprocess.Popen(shlex.split(command),
stdout=log_handle,
stderr=subprocess.STDOUT,
bufsize=1)
self.proc_rcv.append(proc)
except OSError as e:
print "Run::create_receivers - ERROR: Execution failed: %s", e
time.sleep(0.5)
def create_senders(self):
# Keep track of sender processes and handles: needed for final celeanup
self.proc_snd = []
self.hand_snd = []
# If an ADCBoard process runs on a node which is different from the Merger
# we create the sending end of a network tunnel
for adc in (self.adcboard_list):
# Check if ADCBoard and Merger run on different nodes
if adc.node_id != self.merger.node_id:
# Define port for network tunnel
port_number = self.base_port_number + adc.board_id
print "Creating sending end of network tunnel for board %d on port %d" % (
adc.board_id, port_number)
# Define log file and open it
log_file = "%s/%s_nc_%d_send.log" % (
self.log_dir, self.log_file_head, port_number)
log_handle = open(log_file, "w")
self.hand_snd.append(log_handle)
# Open sending end of tunnel on DAQ node. Add some code to wait for receiving end to appear before proceeding.
command = "while ! nc -z %s %d ; do sleep 1 ; done ; nc -v --send-only %s %d < %s > /dev/null" % (
self.merger.node_ip, port_number, self.merger.node_ip,
port_number, adc.output_stream_zsup)
#command = "nc -v --udp --send-only %s %d < %s > /dev/null"%(self.merger.node_ip,port_number,adc.output_stream_zsup)
if adc.node_id != 0:
command = "ssh -f -i %s %s '( %s )'" % (
self.ssh_id_file, adc.node_ip, command)
print command
try:
proc = subprocess.Popen(shlex.split(command),
stdout=log_handle,
stderr=subprocess.STDOUT,
bufsize=1)
self.proc_snd.append(proc)
except OSError as e:
print "Run::create_senders - ERROR: Execution failed: %s", e
time.sleep(0.5)
# If the Trigger process runs on a node which is different from the Merger
# we create the sending end of a network tunnel
if self.trigger.node_id != self.merger.node_id:
# Define port for network tunnel
port_number = self.base_port_number + 99
print "Creating sending end of network tunnel for trigger on port %d" % port_number
# Define log file and open it
log_file = "%s/%s_nc_%d_send.log" % (
self.log_dir, self.log_file_head, port_number)
log_handle = open(log_file, "w")
self.hand_snd.append(log_handle)
# Open sending end of tunnel on Trigger node. Add some code to wait for receiving end to appear before proceeding.
#command = "while ! nc -z %s %d ; do sleep 1 ; done ; nc -v --send-only %s %d < %s > /dev/null"%(self.merger.node_ip,port_number,self.merger.node_ip,port_number,self.trigger.output_stream)
command = "nc -v --udp --send-only %s %d < %s > /dev/null" % (
self.merger.node_ip, port_number, self.trigger.output_stream)
if adc.node_id != 0:
command = "ssh -f -i %s %s '( %s )'" % (
self.ssh_id_file, self.trigger.node_ip, command)
print command
try:
proc = subprocess.Popen(shlex.split(command),
stdout=log_handle,
stderr=subprocess.STDOUT,
bufsize=1)
self.proc_snd.append(proc)
except OSError as e:
print "Run::create_senders - ERROR: Execution failed: %s", e
time.sleep(0.5)
def create_merger_input_list(self):
print "Creating merger input list file %s" % self.merger.input_list
f = open(self.merger.input_list, "w")
for adcboard in self.adcboard_list:
f.write("%d %s\n" %
(adcboard.board_id, adcboard.output_stream_zsup))
f.write("%d %s\n" % (99, self.trigger.output_stream))
f.close()
def create_merger_output_list(self):
print "Creating merger output list file %s" % self.merger.output_list
f = open(self.merger.output_list, "w")
for lvl1 in self.level1_list:
f.write("%s\n" % (lvl1.input_stream))
f.close()
def create_level1_output_dirs(self):
for level1 in self.level1_list:
print "Creating output dir %s for level1 %d" % (level1.output_dir,
level1.level1_id)
if level1.node_id == 0:
if not os.path.exists(level1.output_dir):
os.makedirs(level1.output_dir, 0755)
else:
command = "ssh -i %s %s '( mkdir -p %s )'" % (
self.ssh_id_file, level1.node_ip, level1.output_dir)
print command
os.system(command)
def change_setup(self, setup):
# Reset run configuration to its default values
self.set_default_config()
# Read new setup
self.setup = setup
if (self.read_setup() == "error"): return "error"
# Create new set of ADC board processes (DAQ and ZSUP) handlers
self.daq_nodes_id_list = []
for b in self.boardid_list:
print "Run - Configuring ADC board %d" % b
adcboard = ADCBoard(b)
self.configure_adcboard(adcboard)
self.adcboard_list.append(adcboard)
self.daq_nodes_id_list.append(adcboard.node_id)
# Get unique list of DAQ nodes (needed to create start/stop files)
self.daq_nodes_id_list = list(set(self.daq_nodes_id_list))
# Store ip addresses of DAQ nodes in a dictionary
self.daq_nodes_ip_list = {}
for node_id in self.daq_nodes_id_list:
self.daq_nodes_ip_list[node_id] = self.db.get_node_daq_ip(node_id)
# Create new Trigger process handler
self.trigger = Trigger()
self.configure_trigger(self.trigger)
# Create new Merger process handler
self.merger = Merger()
self.configure_merger(self.merger)
# Create new set of Level1 process handlers
for l in range(self.level1_nproc):
print "Run - Configuring Level1 process %d" % l
lvl1_proc = Level1(l)
self.configure_level1(lvl1_proc)
self.level1_list.append(lvl1_proc)
return setup
def configure_adcboard(self, adcboard):
# Configure ADC board DAQ/ZSUP processes after changing setup
# Reset ADC board to default configuration
adcboard.set_default_config()
# Set executable
adcboard.executable = self.daq_executable
# Lock files (will contain PID of processes)
adcboard.lock_file_daq = "%s_b%02d_daq" % (self.lock_file_head,
adcboard.board_id)
adcboard.lock_file_zsup = "%s_b%02d_zsup" % (self.lock_file_head,
adcboard.board_id)
# Control files needed to start/stop DAQ (will disappear)
adcboard.start_file = self.start_file
adcboard.quit_file = self.quit_file
# Status files for initialization
adcboard.initok_file_daq = "%s_b%02d_daq" % (self.initok_file_head,
adcboard.board_id)
adcboard.initok_file_zsup = "%s_b%02d_zsup" % (self.initok_file_head,
adcboard.board_id)
adcboard.initfail_file_daq = "%s_b%02d_daq" % (self.initfail_file_head,
adcboard.board_id)
adcboard.initfail_file_zsup = "%s_b%02d_zsup" % (
self.initfail_file_head, adcboard.board_id)
# Define board connection information
for link in self.board_link_list:
(board, host, port, node) = link
if (int(board) == adcboard.board_id):
adcboard.node_id = self.db.get_node_id(host)
adcboard.node_ip = self.db.get_node_daq_ip(adcboard.node_id)
adcboard.conet2_link = int(port)
adcboard.conet2_slot = int(node)
# Define total DAQ time (default: 0, i.e. run forever)
# In most cases the default is what you want
adcboard.total_daq_time = self.total_daq_time
# Read ADC settings from board setup file
adcboard.read_setup(self.setup)
def configure_trigger(self, trigger):
# Configure Trigger process after changing setup
# Reset Trigger process handler to default configuration
trigger.set_default_config()
# Set executable
trigger.executable = self.trigger_executable
# Lock file (will contain PID of process)
trigger.lock_file = "%s_trigger" % self.lock_file_head
# Control files needed to start/stop trigger generation
trigger.start_file = self.trig_start_file
trigger.quit_file = self.trig_stop_file
# Status files for initialization
trigger.initok_file = "%s_trigger" % self.initok_file_head
trigger.initfail_file = "%s_trigger" % self.initfail_file_head
# Set node where Trigger will run
trigger.node_id = self.db.get_node_id(self.trigger_node)
trigger.node_ip = self.db.get_node_daq_ip(trigger.node_id)
# Define trigger mask to use for this setup
#trigger.trigger_mask = self.trigger_mask
# Define total DAQ time (default: 0, i.e. run forever)
# In most cases the default is what you want
trigger.total_daq_time = self.total_daq_time
# Read Trigger settings from trigger setup file
trigger.read_setup(self.setup)
def configure_merger(self, merger):
# Configure Merger process after changing setup
# Reset Merger process handler to default configuration
merger.set_default_config()
# Set executable
merger.executable = self.merger_executable
def configure_level1(self, level1):
# Configure Level1 processes after changing setup
# Reset Level1 process handler to default configuration
level1.set_default_config()
# Set executable
level1.executable = self.level1_executable
# Set maximum number of events to write in a single file
level1.max_events = self.level1_maxevt
def runconfig_adcboard(self, adcboard):
# Configure ADC board DAQ/ZSUP processes after changing run
adcboard.run_number = self.run_number
if (self.run_type == "FAKE"):
adcboard.process_mode = "FAKE"
else:
adcboard.process_mode = "DAQ"
s_bid = "b%02d" % adcboard.board_id
adcboard.config_file_daq = "%s/%s_%s_daq.cfg" % (
self.config_dir, self.config_file_head, s_bid)
adcboard.config_file_zsup = "%s/%s_%s_zsup.cfg" % (
self.config_dir, self.config_file_head, s_bid)
adcboard.log_file_daq = "%s/%s_%s_daq.log" % (
self.log_dir, self.log_file_head, s_bid)
adcboard.log_file_zsup = "%s/%s_%s_zsup.log" % (
self.log_dir, self.log_file_head, s_bid)
adcboard.output_stream_daq = "%s/%s_%s_daq" % (self.stream_dir,
self.stream_head, s_bid)
adcboard.input_stream_zsup = adcboard.output_stream_daq
adcboard.output_stream_zsup = "%s/%s_%s_zsup" % (
self.stream_dir, self.stream_head, s_bid)
def runconfig_trigger(self, trigger):
# Configure Trigger process after changing run
trigger.run_number = self.run_number
trigger.config_file = "%s/%s_trigger.cfg" % (self.config_dir,
self.config_file_head)
trigger.log_file = "%s/%s_trigger.log" % (self.log_dir,
self.log_file_head)
trigger.output_stream = "%s/%s_trigger" % (self.stream_dir,
self.stream_head)
def runconfig_merger(self, merger):
# Configure Merger process after changing run
merger.run_number = self.run_number
# Get node_id and node_ip from DB
merger.node_id = self.db.get_node_id(self.merger_node)
merger.node_ip = self.db.get_node_daq_ip(merger.node_id)
merger.config_file = "%s/%s_merger.cfg" % (self.config_dir,
self.config_file_head)
merger.log_file = "%s/%s_merger.log" % (self.log_dir,
self.log_file_head)
merger.input_list = "%s/%s_merger_input.list" % (self.config_dir,
self.config_file_head)
merger.output_list = "%s/%s_merger_output.list" % (
self.config_dir, self.config_file_head)
def runconfig_level1(self, level1):
# Configure Level1 processes after changing run
level1.run_number = self.run_number
# Get node_id and node_ip from DB using Merger node
level1.node_id = self.db.get_node_id(self.merger_node)
level1.node_ip = self.db.get_node_daq_ip(level1.node_id)
s_lid = "lvl1_%02d" % level1.level1_id
level1.config_file = "%s/%s_%s.cfg" % (self.config_dir,
self.config_file_head, s_lid)
level1.log_file = "%s/%s_%s.log" % (self.log_dir, self.log_file_head,
s_lid)
level1.input_stream = "%s/%s_%s" % (self.stream_dir, self.stream_head,
s_lid)
level1.output_dir = self.rawdata_dir
level1.output_header = "%s_%s" % (self.rawdata_head, s_lid)
def start(self):
# Create the "start the run" tag file on all DAQ nodes
print "Starting DAQs"
for node_id in self.daq_nodes_id_list:
if (node_id == 0):
open(self.start_file, 'w').close()
else:
command = "ssh -n -i %s %s '( touch %s )'" % (
self.ssh_id_file, self.daq_nodes_ip_list[node_id],
self.start_file)
print command
os.system(command)
# Wait 5sec before enabling triggers
time.sleep(5)
# Enable triggers
print "Enabling triggers"
if (self.trigger.node_id) == 0:
open(self.trig_start_file, 'w').close()
else:
command = "ssh -n -i %s %s '( touch %s )'" % (
self.ssh_id_file, self.trigger.node_ip, self.trig_start_file)
print command
os.system(command)
# Update run status in DB
if (self.run_number):
self.db.set_run_time_start(self.run_number, self.db.now_str())
self.db.set_run_status(self.run_number,
self.db.DB_RUN_STATUS_RUNNING)
def stop(self):
# Disable triggers
print "Disabling triggers"
if (self.trigger.node_id) == 0:
open(self.trig_stop_file, 'w').close()
else:
command = "ssh -n -i %s %s '( touch %s )'" % (
self.ssh_id_file, self.trigger.node_ip, self.trig_stop_file)
print command
os.system(command)
# Wait 1sec before telling all processes to stop
time.sleep(1)
# Create the "stop the run" tag file on all DAQ nodes
for node_id in self.daq_nodes_id_list:
if (node_id == 0):
open(self.quit_file, 'w').close()
else:
command = "ssh -n -i %s %s '( touch %s )'" % (
self.ssh_id_file, self.daq_nodes_ip_list[node_id],
self.quit_file)
print command
os.system(command)
# Write run name to last_run file for monitoring
with open(self.last_run_file, "w") as lf:
lf.write("%s\n" % self.run_name)
# Finalize run in DB
if (self.run_number):
self.db.set_run_status(self.run_number, self.final_status)
self.db.set_run_time_stop(self.run_number, self.db.now_str())
self.db.set_run_comment_end(self.run_number, self.db.now_str(),
self.run_comment_end)
def clean_up(self):
# Clean up control directories at end of run
print "Cleaning up run directories"
# Remove the "start/stop the run" tag files on all DAQ nodes
for node_id in self.daq_nodes_id_list:
if (node_id == 0):
if (os.path.exists(self.start_file)):
os.remove(self.start_file)
if (os.path.exists(self.quit_file)): os.remove(self.quit_file)
else:
command = "ssh -n -i %s %s '( rm -f %s %s )'" % (
self.ssh_id_file, self.daq_nodes_ip_list[node_id],
self.start_file, self.quit_file)
print command
os.system(command)
# Remove the "start/stop the triggers" tag files on Trigger node
if (self.trigger.node_id == 0):
if (os.path.exists(self.trig_start_file)):
os.remove(self.trig_start_file)
if (os.path.exists(self.trig_stop_file)):
os.remove(self.trig_stop_file)
else:
command = "ssh -n -i %s %s '( rm -f %s %s )'" % (
self.ssh_id_file, self.trigger.node_ip, self.trig_start_file,
self.trig_stop_file)
print command
os.system(command)
# Remove initok/initfail files for all ADC nodes
for adc in (self.adcboard_list):
if (adc.node_id == 0):
if (os.path.exists(adc.initok_file_daq)):
os.remove(adc.initok_file_daq)
if (os.path.exists(adc.initok_file_zsup)):
os.remove(adc.initok_file_zsup)
if (os.path.exists(adc.initfail_file_daq)):
os.remove(adc.initfail_file_daq)
if (os.path.exists(adc.initfail_file_zsup)):
os.remove(adc.initfail_file_zsup)
else:
command = "ssh -n -i %s %s '( rm -f %s %s %s %s)'" % (
self.ssh_id_file, adc.node_ip, adc.initok_file_daq,
adc.initok_file_zsup, adc.initfail_file_daq,
adc.initfail_file_zsup)
print command
os.system(command)
# Remove initok/initfail files on Trigger node
if (self.trigger.node_id == 0):
if (os.path.exists(self.trigger.initok_file)):
os.remove(self.trigger.initok_file)
if (os.path.exists(self.trigger.initfail_file)):
os.remove(self.trigger.initfail_file)
else:
command = "ssh -n -i %s %s '( rm -f %s %s)'" % (
self.ssh_id_file, self.trigger.node_ip,
self.trigger.initok_file, self.trigger.initfail_file)
print command
os.system(command)
# Stop all receiving nc processes on the merger nodes (clumsy but could not find another way)
if (self.merger.node_id == 0):
for proc in self.proc_rcv:
proc.terminate()
else:
command = "ssh -i %s %s '( ps -fu %s | grep recv-only | grep -v bash | grep -v grep | awk \"{print \$2}\" | xargs kill )'" % (
self.ssh_id_file, self.merger.node_ip, os.getenv(
'USER', "daq"))
print command
os.system(command)
# Now we can clean up all nc processes
for proc in self.proc_snd:
if proc.poll() != None:
proc.wait()
else:
print "Run::clean_up - Problem closing sending nc process"
for proc in self.proc_rcv:
if proc.poll() != None:
proc.wait()
else:
print "Run::clean_up - Problem closing receiving nc process"
# Close all receiving/sending nc log files
for handler in self.hand_rcv:
handler.close()
for handler in self.hand_snd:
handler.close()
def create_run(self):
if (self.run_number):
# Connect to DB
db = PadmeDB()
db.create_run(self.run_number, self.run_type, self.run_user,
self.run_comment)
# Write run configuration parameters to DB
db.add_cfg_para(self.run_number, "setup", self.setup)
s_board_list = ""
for s_bid in self.s_boards:
s_board_list += " " + s_bid
db.add_cfg_para(self.run_number, "board_list", s_board_list)
db.add_cfg_para(self.run_number, "config_dir", self.config_dir)
db.add_cfg_para(self.run_number, "config_file", self.config_file)
db.add_cfg_para(self.run_number, "config_file_head",
self.config_file_head)
db.add_cfg_para(self.run_number, "log_dir", self.log_dir)
db.add_cfg_para(self.run_number, "log_file_head",
self.log_file_head)
db.add_cfg_para(self.run_number, "start_file", self.start_file)
db.add_cfg_para(self.run_number, "quit_file", self.quit_file)
db.add_cfg_para(self.run_number, "initok_file_head",
self.initok_file_head)
db.add_cfg_para(self.run_number, "initfail_file_head",
self.initfail_file_head)
db.add_cfg_para(self.run_number, "lock_file_head",
self.lock_file_head)
#db.add_cfg_para(self.run_number,"db_file", self.db_file)
db.add_cfg_para(self.run_number, "data_dir", self.data_dir)
db.add_cfg_para(self.run_number, "data_file_head",
self.data_file_head)
db.add_cfg_para(self.run_number, "total_daq_time",
self.total_daq_time)
del db
