def create_run(self): if (self.run_number): # Connect to DB db = PadmeDB() db.create_run(self.run_number,self.run_type,self.run_user,self.run_comment) # Write run configuration parameters to DB db.add_cfg_para(self.run_number,"setup", self.setup) s_board_list = "" for s_bid in self.s_boards: s_board_list += " "+s_bid db.add_cfg_para(self.run_number,"board_list", s_board_list) db.add_cfg_para(self.run_number,"config_dir", self.config_dir) db.add_cfg_para(self.run_number,"config_file", self.config_file) db.add_cfg_para(self.run_number,"config_file_head", self.config_file_head) db.add_cfg_para(self.run_number,"log_dir", self.log_dir) db.add_cfg_para(self.run_number,"log_file_head", self.log_file_head) db.add_cfg_para(self.run_number,"start_file", self.start_file) db.add_cfg_para(self.run_number,"quit_file", self.quit_file) db.add_cfg_para(self.run_number,"initok_file_head", self.lock_file_head) db.add_cfg_para(self.run_number,"initfail_file_head", self.lock_file_head) db.add_cfg_para(self.run_number,"lock_file_head", self.lock_file_head) #db.add_cfg_para(self.run_number,"db_file", self.db_file) db.add_cfg_para(self.run_number,"data_dir", self.data_dir) db.add_cfg_para(self.run_number,"data_file_head", self.data_file_head) db.add_cfg_para(self.run_number,"total_daq_time", self.total_daq_time) del db
def create_run(self): if (self.run_number): # Connect to DB db = PadmeDB() db.create_run(self.run_number, self.run_type, self.run_user, self.run_comment) # Write run configuration parameters to DB db.add_cfg_para(self.run_number, "setup", self.setup) s_board_list = "" for s_bid in self.s_boards: s_board_list += " " + s_bid db.add_cfg_para(self.run_number, "board_list", s_board_list) db.add_cfg_para(self.run_number, "config_dir", self.config_dir) db.add_cfg_para(self.run_number, "config_file", self.config_file) db.add_cfg_para(self.run_number, "config_file_head", self.config_file_head) db.add_cfg_para(self.run_number, "log_dir", self.log_dir) db.add_cfg_para(self.run_number, "log_file_head", self.log_file_head) db.add_cfg_para(self.run_number, "start_file", self.start_file) db.add_cfg_para(self.run_number, "quit_file", self.quit_file) db.add_cfg_para(self.run_number, "initok_file_head", self.initok_file_head) db.add_cfg_para(self.run_number, "initfail_file_head", self.initfail_file_head) db.add_cfg_para(self.run_number, "lock_file_head", self.lock_file_head) #db.add_cfg_para(self.run_number,"db_file", self.db_file) db.add_cfg_para(self.run_number, "data_dir", self.data_dir) db.add_cfg_para(self.run_number, "data_file_head", self.data_file_head) db.add_cfg_para(self.run_number, "total_daq_time", self.total_daq_time) del db
class Run: def __init__(self): # Get account under which the RunControl runs self.user_account = os.getenv('USER', "daq") # Get location of DAQ main directory from PADME_DAQ_DIR # Default to current dir if not set self.daq_dir = os.getenv('PADME_DAQ_DIR', ".") # Get base port number for network tunnels from PADME_RC_TUNNEL_BASE_PORT self.base_port_number = int( os.getenv('PADME_RC_TUNNEL_BASE_PORT', "31400")) # Define id file for passwordless ssh command execution self.ssh_id_file = "%s/.ssh/id_rsa_daq" % os.getenv('HOME', "~") # Get location of padme-fw software from PADME # Default to PADME_DAQ_DIR/padme-fw if not set self.padme_fw = os.getenv('PADME', "%s/padme-fw" % self.daq_dir) # Define executables to use in production self.daq_executable = "%s/PadmeDAQ/PadmeDAQ.exe" % self.padme_fw self.merger_executable = "%s/Level1/PadmeMerger.exe" % self.padme_fw self.level1_executable = "%s/Level1/PadmeLevel1.exe" % self.padme_fw self.trigger_executable = "%s/PadmeTrig/PadmeTrig.exe" % self.padme_fw # Define directory containing setup subdirectories self.setup_root_dir = "%s/setup" % self.daq_dir # Define files where the current and last run name will be written self.current_run_file = "%s/run/current_run" % self.daq_dir self.last_run_file = "%s/run/last_run" % self.daq_dir # Define directory containing rawdata directories for each run self.rawdata_root_dir = self.daq_dir + "/local/rawdata" # Define name and position of control files self.control_dir = self.daq_dir + "/local/run" self.start_file = self.control_dir + "/start" self.quit_file = self.control_dir + "/quit" self.trig_start_file = self.control_dir + "/start_trig" self.trig_stop_file = self.control_dir + "/stop_trig" self.initok_file_head = self.control_dir + "/initok" self.initfail_file_head = self.control_dir + "/initfail" self.lock_file_head = self.control_dir + "/lock" # Connect to database services self.db = PadmeDB() # Do not define a default setup self.setup = "" # Run final status defaults to 3 (stopped normally) self.final_status = 3 self.set_default_config() def change_run(self): # Check if requested run number was not used before # Saves the day if more than one RunControl program is running at the same time (DON'T DO THAT!!!) if (self.run_number): run_is_in_db = self.db.is_run_in_db(self.run_number) if (run_is_in_db): print "Run::change_run - ERROR - Run %d is already in the DB: cannot use it again" % self.run_number print "Please check if someone else is using this RunControl before retrying" #self.send_answer("error_init") return False # Define run name using run number and start time self.run_name = "run_%7.7d_%s" % ( self.run_number, time.strftime("%Y%m%d_%H%M%S", time.gmtime())) # Write run name to current_run file for monitoring with open(self.current_run_file, "w") as lf: lf.write("%s\n" % self.run_name) self.run_dir = self.daq_dir + "/runs/" + self.run_name self.config_dir = "%s/cfg" % self.run_dir self.config_file = "%s.cfg" % self.run_name self.config_file_head = self.run_name self.log_dir = "%s/log" % self.run_dir self.log_file_head = self.run_name self.stream_dir = "%s/local/streams/%s" % (self.daq_dir, self.run_name) self.stream_head = self.run_name self.rawdata_dir = "%s/%s" % (self.rawdata_root_dir, self.run_name) self.rawdata_head = self.run_name # Make sure Merger runs on a different node after each run self.merger_node = self.next_merger_node() # Configure Merger for this run self.runconfig_merger(self.merger) # Configure Trigger for this run self.runconfig_trigger(self.trigger) # Configure ADC boards for this run for adcboard in self.adcboard_list: #print "--- Configuring for run board %d"%adcboard.board_id self.runconfig_adcboard(adcboard) # Configure Level1 processes for this run for level1 in self.level1_list: self.runconfig_level1(level1) # If this is a real run, create it in the DB if (self.run_number): print "Creating Run %d structure in DB" % self.run_number if self.create_run_in_db() == "error": print "Run::change_run - ERROR - Cannot create Run in the DB" return False return True def next_merger_node(self): # If merger node is currently not defined, set it to first in list or localhost if not self.merger_node: if self.merger_node_list: return self.merger_node_list[0] else: return "localhost" # If the list of nodes was not defined, keep using always the same node if not self.merger_node_list: return self.merger_node # Find current node in node list and use next one use_next = False for node in self.merger_node_list: if use_next: return node if node == self.merger_node: use_next = True # Current node was the last in the list, so we restart with the first node in the list if use_next: return self.merger_node_list[0] # Handle misconfigurations (should never happen) print "Run::next_merger_node - WARNING: current merger host %s is not in the merger node list %s" % ( self.merger_node, self.merger_node_list) print " Will use first node in list: %s" % self.merger_node_list[ 0] return self.merger_node_list[0] def set_default_config(self): # Clean up Run configuration and set all run parameters to default self.adcboard_list = [] self.trigger = None self.merger = None self.level1_list = [] self.run_number = 0 self.run_name = "run_%7.7d_%s" % ( self.run_number, time.strftime("%Y%m%d_%H%M%S", time.gmtime())) self.run_type = "TEST" self.run_user = "******" self.run_comment_start = "Generic start of run" self.run_comment_end = "Generic end of run" self.run_dir = "%s/runs/%s" % (self.daq_dir, self.run_name) self.config_dir = "%s/cfg" % self.run_dir self.config_file = "%s.cfg" % self.run_name self.config_file_head = self.run_name self.log_dir = "%s/log" % self.run_dir self.log_file_head = self.run_name self.stream_dir = "%s/local/streams/%s" % (self.daq_dir, self.run_name) self.stream_head = self.run_name self.rawdata_dir = "%s/%s" % (self.rawdata_root_dir, self.run_name) self.rawdata_head = self.run_name self.trigger_node = "localhost" self.merger_node = "localhost" self.merger_node_list = [] self.level1_nproc = 1 self.level1_maxevt = 10000 self.total_daq_time = 0 def read_setup(self): # Define regular expressions used in file parsing re_empty = re.compile("^\s*$") re_comment = re.compile("^\s*#") re_param = re.compile("^\s*(\w+)\s+(.+?)\s*$") re_boardid = re.compile("\d+") re_board_link = re.compile("^\s*(\d+)\s+(\S+)\s+(\d+)\s+(\d+)\s*$") # Read general run configuration from file setup_file = "%s/%s/run.cfg" % (self.setup_root_dir, self.setup) if (not os.path.isfile(setup_file)): print "Run - ERROR: setup file %s not found" % setup_file return "error" f = open(setup_file) self.boardid_list = [] self.board_link_list = [] for l in f: if (re_empty.search(l) or re_comment.search(l)): continue m = re_param.search(l) if (m): (p_name, p_value) = m.group(1, 2) if (p_name == "boardid_list"): # Get sorted list of board ids while removing duplicates s_boards = sorted(list(set(re_boardid.findall(p_value))), key=int) for s_bid in s_boards: # Convert to int as findall returns strings self.boardid_list.append(int(s_bid)) elif (p_name == "board_link"): mm = re_board_link.search(p_value) if (mm): self.board_link_list.append(mm.group(1, 2, 3, 4)) else: print "Run - WARNING: unable to decode board_link parameter while reading setup file %s" % ( setup_file, ) print l elif (p_name == "total_daq_time"): self.total_daq_time = int(p_value) elif (p_name == "trigger_node"): self.trigger_node = p_value #elif (p_name == "trigger_mask"): # self.trigger_mask = p_value elif (p_name == "merger_node"): self.merger_node = p_value elif (p_name == "merger_node_list"): self.merger_node_list = p_value.split() elif (p_name == "level1_nproc"): self.level1_nproc = int(p_value) elif (p_name == "level1_maxevt"): self.level1_maxevt = int(p_value) else: print "Run - WARNING: unknown parameter %s found while reading setup file %s" % ( p_name, setup_file) else: print "Run - WARNING: unknown line format found while reading setup file %s" % ( p_name, setup_file) print l f.close() return "ok" def config_list(self): cfg_list = [] cfg_list.append(["user_account", self.user_account]) cfg_list.append(["daq_dir", self.daq_dir]) cfg_list.append(["base_port_number", self.base_port_number]) cfg_list.append(["ssh_id_file", self.ssh_id_file]) cfg_list.append(["daq_executable", self.daq_executable]) cfg_list.append(["trigger_executable", self.trigger_executable]) cfg_list.append(["merger_executable", self.merger_executable]) cfg_list.append(["level1_executable", self.level1_executable]) cfg_list.append(["start_file", self.start_file]) cfg_list.append(["quit_file", self.quit_file]) cfg_list.append(["trig_start_file", self.trig_start_file]) cfg_list.append(["trig_stop_file", self.trig_stop_file]) cfg_list.append(["initok_file_head", self.initok_file_head]) cfg_list.append(["initfail_file_head", self.initfail_file_head]) cfg_list.append(["lock_file_head", self.lock_file_head]) cfg_list.append(["rawdata_dir", self.rawdata_dir]) cfg_list.append(["run_number", str(self.run_number)]) cfg_list.append(["run_name", self.run_name]) cfg_list.append(["run_dir", self.run_dir]) cfg_list.append(["run_type", self.run_type]) cfg_list.append(["run_user", self.run_user]) cfg_list.append(["run_comment_start", self.run_comment_start]) cfg_list.append(["setup", self.setup]) s_board_list = "" for b in self.boardid_list: if (s_board_list): s_board_list += " %d" % b else: s_board_list = "%d" % b cfg_list.append(["board_list", s_board_list]) for b in self.boardid_list: for link in self.board_link_list: (board, host, port, node) = link if b == int(board): board_link = "%s %s %s %s" % (board, host, port, node) cfg_list.append([ "board_link", "%s %s %s %s" % (board, host, port, node) ]) cfg_list.append(["config_dir", self.config_dir]) cfg_list.append(["config_file", self.config_file]) cfg_list.append(["config_file_head", self.config_file_head]) cfg_list.append(["log_dir", self.log_dir]) cfg_list.append(["log_file_head", self.log_file_head]) cfg_list.append(["stream_dir", self.stream_dir]) cfg_list.append(["stream_head", self.stream_head]) cfg_list.append(["rawdata_dir", self.rawdata_dir]) cfg_list.append(["rawdata_head", self.rawdata_head]) cfg_list.append(["trigger_node", self.trigger_node]) if self.merger_node: cfg_list.append(["merger_node", self.merger_node]) if self.merger_node_list: cfg_list.append( ["merger_node_list", " ".join(self.merger_node_list)]) cfg_list.append(["level1_nproc", str(self.level1_nproc)]) cfg_list.append(["level1_maxevt", str(self.level1_maxevt)]) cfg_list.append(["total_daq_time", str(self.total_daq_time)]) return cfg_list def format_config(self): cfgstring = "" for cfg in self.config_list(): cfgstring += "%-30s %s\n" % (cfg[0], cfg[1]) return cfgstring def create_run_in_db(self): # Create run in DB self.db.create_run(self.run_number, self.run_name, self.run_type) self.db.set_run_time_create(self.run_number, self.db.now_str()) self.db.set_run_user(self.run_number, self.run_user) self.db.set_run_comment_start(self.run_number, self.db.now_str(), self.run_comment_start) # Add all configuration parameters for cfg in self.config_list(): self.db.add_cfg_para_run(self.run_number, cfg[0], cfg[1]) # Create board structures in DB for adc in (self.adcboard_list): if adc.create_proc_daq() == "error": print "Run::create_run - ERROR - Cannot create DAQ process for board %d in the DB" % adc.board_id return "error" if adc.create_proc_zsup() == "error": print "Run::create_run - ERROR - Cannot create ZSUP process for board %d in the DB" % adc.board_id return "error" # Create Trigger structure in DB if self.trigger.create_trigger() == "error": print "Run::create_run - ERROR - Cannot create Trigger process in the DB" return "error" # Create Merger structure in DB if self.merger.create_merger() == "error": print "Run::create_run - ERROR - Cannot create Merger process in the DB" return "error" # Create Level1 structures in DB for lvl1 in (self.level1_list): if lvl1.create_level1() == "error": print "Run::create_run - ERROR - Cannot create Level1 process for level1 %d in the DB" % lvl1.level1_id return "error" return "ok" def create_log_dir(self): # Create log directory for this run (make sure the full tree is there) if not os.path.exists(self.log_dir): os.makedirs(self.log_dir, 0755) def write_config(self): # Create config directory for this run (make sure the full tree is there) if not os.path.exists(self.config_dir): os.makedirs(self.config_dir, 0755) print "Writing configuration file %s" % self.config_file f = open(self.config_dir + "/" + self.config_file, "w") f.write(self.format_config()) f.close() print "Writing configuration file %s for Merger" % self.merger.config_file self.merger.write_config() print "Writing configuration file %s for Trigger" % self.trigger.config_file self.trigger.write_config() for adc in (self.adcboard_list): print "Writing configuration files %s and %s for ADC board %d" % ( adc.config_file_daq, adc.config_file_zsup, adc.board_id) adc.write_config() for lvl1 in (self.level1_list): print "Writing configuration files %s for Level1 %d" % ( lvl1.config_file, lvl1.level1_id) lvl1.write_config() def print_config(self): print self.format_config() def create_fifos(self): # Create stream directory and fifo file for Trigger if self.trigger.node_id == 0: if not os.path.exists(self.stream_dir): os.makedirs(self.stream_dir, 0755) os.mkfifo(self.trigger.output_stream) else: command = "ssh -i %s %s '( mkdir -p %s ; mkfifo %s )'" % ( self.ssh_id_file, self.trigger.node_ip, self.stream_dir, self.trigger.output_stream) print command os.system(command) # Create stream directories and fifo files for ADCBoards for adc in (self.adcboard_list): if adc.node_id == 0: if not os.path.exists(self.stream_dir): os.makedirs(self.stream_dir, 0755) os.mkfifo(adc.output_stream_daq) os.mkfifo(adc.output_stream_zsup) else: command = "ssh -i %s %s '( mkdir -p %s ; mkfifo %s %s )'" % ( self.ssh_id_file, adc.node_ip, self.stream_dir, adc.output_stream_daq, adc.output_stream_zsup) print command os.system(command) # Create stream directory on Merger if self.merger.node_id == 0: if not os.path.exists(self.stream_dir): os.makedirs(self.stream_dir, 0755) else: command = "ssh -i %s %s '( mkdir -p %s )'" % ( self.ssh_id_file, self.merger.node_ip, self.stream_dir) print command os.system(command) # Create fifo files to connect Merger to Level1 processes for lvl1 in (self.level1_list): if lvl1.node_id == 0: os.mkfifo(lvl1.input_stream) else: command = "ssh -i %s %s '( mkfifo %s )'" % ( self.ssh_id_file, lvl1.node_ip, lvl1.input_stream) print command os.system(command) # If Trigger process runs on a node which is different from the Merger # we must replicate the Trigger FIFO file on the Merger if self.trigger.node_id != self.merger.node_id: # Duplicate Trigger FIFO file on Merger node if self.merger.node_id == 0: os.mkfifo(self.trigger.output_stream) else: command = "ssh -n -i %s %s '( mkfifo %s )'" % ( self.ssh_id_file, self.merger.node_ip, self.trigger.output_stream) print command os.system(command) # If an ADCBoard process runs on a node which is different from the Merger # we must replicate the ZSUP FIFO file on the Merger if self.merger.node_id == 0: for adc in (self.adcboard_list): if adc.node_id != self.merger.node_id: os.mkfifo(adc.output_stream_zsup) else: # More efficient if we give a single (long) ssh command stream_list = "" for adc in (self.adcboard_list): if adc.node_id != self.merger.node_id: stream_list += " %s" % adc.output_stream_zsup if stream_list: command = "ssh -n -i %s %s '( mkfifo %s )'" % ( self.ssh_id_file, self.merger.node_ip, stream_list) print command os.system(command) def create_receivers(self): # Keep track of receiver processes and handles: needed for final celeanup self.proc_rcv = [] self.hand_rcv = [] # If an ADCBoard process runs on a node which is different from the Merger # we create the receiving end of a network tunnel for adc in (self.adcboard_list): # Check if ADCBoard and Merger run on different nodes if adc.node_id != self.merger.node_id: # Define port for network tunnel port_number = self.base_port_number + adc.board_id print "Creating receiving end of network tunnel for board %d on port %d" % ( adc.board_id, port_number) # Define log file and open it log_file = "%s/%s_nc_%d_recv.log" % ( self.log_dir, self.log_file_head, port_number) log_handle = open(log_file, "w") self.hand_rcv.append(log_handle) # Open receiving end of tunnel on Merger node command = "nc -l -k -v --recv-only %s %d > %s < /dev/zero" % ( self.merger.node_ip, port_number, adc.output_stream_zsup) #command = "nc --udp -l -v --recv-only %s %d > %s < /dev/zero"%(self.merger.node_ip,port_number,adc.output_stream_zsup) if self.merger.node_id != 0: command = "ssh -f -i %s %s '( %s )'" % ( self.ssh_id_file, self.merger.node_ip, command) print command try: proc = subprocess.Popen(shlex.split(command), stdout=log_handle, stderr=subprocess.STDOUT, bufsize=1) self.proc_rcv.append(proc) except OSError as e: print "Run::create_receivers - ERROR: Execution failed: %s", e time.sleep(0.5) # If the Trigger process runs on a node which is different from the Merger # we create the receiving end of a network tunnel if self.trigger.node_id != self.merger.node_id: # Define port for network tunnel port_number = self.base_port_number + 99 print "Creating receiving end of network tunnel for trigger on port %d" % port_number # Define log file and open it log_file = "%s/%s_nc_%d_recv.log" % ( self.log_dir, self.log_file_head, port_number) log_handle = open(log_file, "w") self.hand_rcv.append(log_handle) # Open receiving end of tunnel on Merger node #command = "nc -l -k -v --recv-only %s %d > %s < /dev/zero"%(self.merger.node_ip,port_number,self.trigger.output_stream) command = "nc --udp -l -v --recv-only %s %d > %s < /dev/zero" % ( self.merger.node_ip, port_number, self.trigger.output_stream) if self.merger.node_id != 0: command = "ssh -f -i %s %s '( %s )'" % ( self.ssh_id_file, self.merger.node_ip, command) print command try: proc = subprocess.Popen(shlex.split(command), stdout=log_handle, stderr=subprocess.STDOUT, bufsize=1) self.proc_rcv.append(proc) except OSError as e: print "Run::create_receivers - ERROR: Execution failed: %s", e time.sleep(0.5) def create_senders(self): # Keep track of sender processes and handles: needed for final celeanup self.proc_snd = [] self.hand_snd = [] # If an ADCBoard process runs on a node which is different from the Merger # we create the sending end of a network tunnel for adc in (self.adcboard_list): # Check if ADCBoard and Merger run on different nodes if adc.node_id != self.merger.node_id: # Define port for network tunnel port_number = self.base_port_number + adc.board_id print "Creating sending end of network tunnel for board %d on port %d" % ( adc.board_id, port_number) # Define log file and open it log_file = "%s/%s_nc_%d_send.log" % ( self.log_dir, self.log_file_head, port_number) log_handle = open(log_file, "w") self.hand_snd.append(log_handle) # Open sending end of tunnel on DAQ node. Add some code to wait for receiving end to appear before proceeding. command = "while ! nc -z %s %d ; do sleep 1 ; done ; nc -v --send-only %s %d < %s > /dev/null" % ( self.merger.node_ip, port_number, self.merger.node_ip, port_number, adc.output_stream_zsup) #command = "nc -v --udp --send-only %s %d < %s > /dev/null"%(self.merger.node_ip,port_number,adc.output_stream_zsup) if adc.node_id != 0: command = "ssh -f -i %s %s '( %s )'" % ( self.ssh_id_file, adc.node_ip, command) print command try: proc = subprocess.Popen(shlex.split(command), stdout=log_handle, stderr=subprocess.STDOUT, bufsize=1) self.proc_snd.append(proc) except OSError as e: print "Run::create_senders - ERROR: Execution failed: %s", e time.sleep(0.5) # If the Trigger process runs on a node which is different from the Merger # we create the sending end of a network tunnel if self.trigger.node_id != self.merger.node_id: # Define port for network tunnel port_number = self.base_port_number + 99 print "Creating sending end of network tunnel for trigger on port %d" % port_number # Define log file and open it log_file = "%s/%s_nc_%d_send.log" % ( self.log_dir, self.log_file_head, port_number) log_handle = open(log_file, "w") self.hand_snd.append(log_handle) # Open sending end of tunnel on Trigger node. Add some code to wait for receiving end to appear before proceeding. #command = "while ! nc -z %s %d ; do sleep 1 ; done ; nc -v --send-only %s %d < %s > /dev/null"%(self.merger.node_ip,port_number,self.merger.node_ip,port_number,self.trigger.output_stream) command = "nc -v --udp --send-only %s %d < %s > /dev/null" % ( self.merger.node_ip, port_number, self.trigger.output_stream) if adc.node_id != 0: command = "ssh -f -i %s %s '( %s )'" % ( self.ssh_id_file, self.trigger.node_ip, command) print command try: proc = subprocess.Popen(shlex.split(command), stdout=log_handle, stderr=subprocess.STDOUT, bufsize=1) self.proc_snd.append(proc) except OSError as e: print "Run::create_senders - ERROR: Execution failed: %s", e time.sleep(0.5) def create_merger_input_list(self): print "Creating merger input list file %s" % self.merger.input_list f = open(self.merger.input_list, "w") for adcboard in self.adcboard_list: f.write("%d %s\n" % (adcboard.board_id, adcboard.output_stream_zsup)) f.write("%d %s\n" % (99, self.trigger.output_stream)) f.close() def create_merger_output_list(self): print "Creating merger output list file %s" % self.merger.output_list f = open(self.merger.output_list, "w") for lvl1 in self.level1_list: f.write("%s\n" % (lvl1.input_stream)) f.close() def create_level1_output_dirs(self): for level1 in self.level1_list: print "Creating output dir %s for level1 %d" % (level1.output_dir, level1.level1_id) if level1.node_id == 0: if not os.path.exists(level1.output_dir): os.makedirs(level1.output_dir, 0755) else: command = "ssh -i %s %s '( mkdir -p %s )'" % ( self.ssh_id_file, level1.node_ip, level1.output_dir) print command os.system(command) def change_setup(self, setup): # Reset run configuration to its default values self.set_default_config() # Read new setup self.setup = setup if (self.read_setup() == "error"): return "error" # Create new set of ADC board processes (DAQ and ZSUP) handlers self.daq_nodes_id_list = [] for b in self.boardid_list: print "Run - Configuring ADC board %d" % b adcboard = ADCBoard(b) self.configure_adcboard(adcboard) self.adcboard_list.append(adcboard) self.daq_nodes_id_list.append(adcboard.node_id) # Get unique list of DAQ nodes (needed to create start/stop files) self.daq_nodes_id_list = list(set(self.daq_nodes_id_list)) # Store ip addresses of DAQ nodes in a dictionary self.daq_nodes_ip_list = {} for node_id in self.daq_nodes_id_list: self.daq_nodes_ip_list[node_id] = self.db.get_node_daq_ip(node_id) # Create new Trigger process handler self.trigger = Trigger() self.configure_trigger(self.trigger) # Create new Merger process handler self.merger = Merger() self.configure_merger(self.merger) # Create new set of Level1 process handlers for l in range(self.level1_nproc): print "Run - Configuring Level1 process %d" % l lvl1_proc = Level1(l) self.configure_level1(lvl1_proc) self.level1_list.append(lvl1_proc) return setup def configure_adcboard(self, adcboard): # Configure ADC board DAQ/ZSUP processes after changing setup # Reset ADC board to default configuration adcboard.set_default_config() # Set executable adcboard.executable = self.daq_executable # Lock files (will contain PID of processes) adcboard.lock_file_daq = "%s_b%02d_daq" % (self.lock_file_head, adcboard.board_id) adcboard.lock_file_zsup = "%s_b%02d_zsup" % (self.lock_file_head, adcboard.board_id) # Control files needed to start/stop DAQ (will disappear) adcboard.start_file = self.start_file adcboard.quit_file = self.quit_file # Status files for initialization adcboard.initok_file_daq = "%s_b%02d_daq" % (self.initok_file_head, adcboard.board_id) adcboard.initok_file_zsup = "%s_b%02d_zsup" % (self.initok_file_head, adcboard.board_id) adcboard.initfail_file_daq = "%s_b%02d_daq" % (self.initfail_file_head, adcboard.board_id) adcboard.initfail_file_zsup = "%s_b%02d_zsup" % ( self.initfail_file_head, adcboard.board_id) # Define board connection information for link in self.board_link_list: (board, host, port, node) = link if (int(board) == adcboard.board_id): adcboard.node_id = self.db.get_node_id(host) adcboard.node_ip = self.db.get_node_daq_ip(adcboard.node_id) adcboard.conet2_link = int(port) adcboard.conet2_slot = int(node) # Define total DAQ time (default: 0, i.e. run forever) # In most cases the default is what you want adcboard.total_daq_time = self.total_daq_time # Read ADC settings from board setup file adcboard.read_setup(self.setup) def configure_trigger(self, trigger): # Configure Trigger process after changing setup # Reset Trigger process handler to default configuration trigger.set_default_config() # Set executable trigger.executable = self.trigger_executable # Lock file (will contain PID of process) trigger.lock_file = "%s_trigger" % self.lock_file_head # Control files needed to start/stop trigger generation trigger.start_file = self.trig_start_file trigger.quit_file = self.trig_stop_file # Status files for initialization trigger.initok_file = "%s_trigger" % self.initok_file_head trigger.initfail_file = "%s_trigger" % self.initfail_file_head # Set node where Trigger will run trigger.node_id = self.db.get_node_id(self.trigger_node) trigger.node_ip = self.db.get_node_daq_ip(trigger.node_id) # Define trigger mask to use for this setup #trigger.trigger_mask = self.trigger_mask # Define total DAQ time (default: 0, i.e. run forever) # In most cases the default is what you want trigger.total_daq_time = self.total_daq_time # Read Trigger settings from trigger setup file trigger.read_setup(self.setup) def configure_merger(self, merger): # Configure Merger process after changing setup # Reset Merger process handler to default configuration merger.set_default_config() # Set executable merger.executable = self.merger_executable def configure_level1(self, level1): # Configure Level1 processes after changing setup # Reset Level1 process handler to default configuration level1.set_default_config() # Set executable level1.executable = self.level1_executable # Set maximum number of events to write in a single file level1.max_events = self.level1_maxevt def runconfig_adcboard(self, adcboard): # Configure ADC board DAQ/ZSUP processes after changing run adcboard.run_number = self.run_number if (self.run_type == "FAKE"): adcboard.process_mode = "FAKE" else: adcboard.process_mode = "DAQ" s_bid = "b%02d" % adcboard.board_id adcboard.config_file_daq = "%s/%s_%s_daq.cfg" % ( self.config_dir, self.config_file_head, s_bid) adcboard.config_file_zsup = "%s/%s_%s_zsup.cfg" % ( self.config_dir, self.config_file_head, s_bid) adcboard.log_file_daq = "%s/%s_%s_daq.log" % ( self.log_dir, self.log_file_head, s_bid) adcboard.log_file_zsup = "%s/%s_%s_zsup.log" % ( self.log_dir, self.log_file_head, s_bid) adcboard.output_stream_daq = "%s/%s_%s_daq" % (self.stream_dir, self.stream_head, s_bid) adcboard.input_stream_zsup = adcboard.output_stream_daq adcboard.output_stream_zsup = "%s/%s_%s_zsup" % ( self.stream_dir, self.stream_head, s_bid) def runconfig_trigger(self, trigger): # Configure Trigger process after changing run trigger.run_number = self.run_number trigger.config_file = "%s/%s_trigger.cfg" % (self.config_dir, self.config_file_head) trigger.log_file = "%s/%s_trigger.log" % (self.log_dir, self.log_file_head) trigger.output_stream = "%s/%s_trigger" % (self.stream_dir, self.stream_head) def runconfig_merger(self, merger): # Configure Merger process after changing run merger.run_number = self.run_number # Get node_id and node_ip from DB merger.node_id = self.db.get_node_id(self.merger_node) merger.node_ip = self.db.get_node_daq_ip(merger.node_id) merger.config_file = "%s/%s_merger.cfg" % (self.config_dir, self.config_file_head) merger.log_file = "%s/%s_merger.log" % (self.log_dir, self.log_file_head) merger.input_list = "%s/%s_merger_input.list" % (self.config_dir, self.config_file_head) merger.output_list = "%s/%s_merger_output.list" % ( self.config_dir, self.config_file_head) def runconfig_level1(self, level1): # Configure Level1 processes after changing run level1.run_number = self.run_number # Get node_id and node_ip from DB using Merger node level1.node_id = self.db.get_node_id(self.merger_node) level1.node_ip = self.db.get_node_daq_ip(level1.node_id) s_lid = "lvl1_%02d" % level1.level1_id level1.config_file = "%s/%s_%s.cfg" % (self.config_dir, self.config_file_head, s_lid) level1.log_file = "%s/%s_%s.log" % (self.log_dir, self.log_file_head, s_lid) level1.input_stream = "%s/%s_%s" % (self.stream_dir, self.stream_head, s_lid) level1.output_dir = self.rawdata_dir level1.output_header = "%s_%s" % (self.rawdata_head, s_lid) def start(self): # Create the "start the run" tag file on all DAQ nodes print "Starting DAQs" for node_id in self.daq_nodes_id_list: if (node_id == 0): open(self.start_file, 'w').close() else: command = "ssh -n -i %s %s '( touch %s )'" % ( self.ssh_id_file, self.daq_nodes_ip_list[node_id], self.start_file) print command os.system(command) # Wait 5sec before enabling triggers time.sleep(5) # Enable triggers print "Enabling triggers" if (self.trigger.node_id) == 0: open(self.trig_start_file, 'w').close() else: command = "ssh -n -i %s %s '( touch %s )'" % ( self.ssh_id_file, self.trigger.node_ip, self.trig_start_file) print command os.system(command) # Update run status in DB if (self.run_number): self.db.set_run_time_start(self.run_number, self.db.now_str()) self.db.set_run_status(self.run_number, self.db.DB_RUN_STATUS_RUNNING) def stop(self): # Disable triggers print "Disabling triggers" if (self.trigger.node_id) == 0: open(self.trig_stop_file, 'w').close() else: command = "ssh -n -i %s %s '( touch %s )'" % ( self.ssh_id_file, self.trigger.node_ip, self.trig_stop_file) print command os.system(command) # Wait 1sec before telling all processes to stop time.sleep(1) # Create the "stop the run" tag file on all DAQ nodes for node_id in self.daq_nodes_id_list: if (node_id == 0): open(self.quit_file, 'w').close() else: command = "ssh -n -i %s %s '( touch %s )'" % ( self.ssh_id_file, self.daq_nodes_ip_list[node_id], self.quit_file) print command os.system(command) # Write run name to last_run file for monitoring with open(self.last_run_file, "w") as lf: lf.write("%s\n" % self.run_name) # Finalize run in DB if (self.run_number): self.db.set_run_status(self.run_number, self.final_status) self.db.set_run_time_stop(self.run_number, self.db.now_str()) self.db.set_run_comment_end(self.run_number, self.db.now_str(), self.run_comment_end) def clean_up(self): # Clean up control directories at end of run print "Cleaning up run directories" # Remove the "start/stop the run" tag files on all DAQ nodes for node_id in self.daq_nodes_id_list: if (node_id == 0): if (os.path.exists(self.start_file)): os.remove(self.start_file) if (os.path.exists(self.quit_file)): os.remove(self.quit_file) else: command = "ssh -n -i %s %s '( rm -f %s %s )'" % ( self.ssh_id_file, self.daq_nodes_ip_list[node_id], self.start_file, self.quit_file) print command os.system(command) # Remove the "start/stop the triggers" tag files on Trigger node if (self.trigger.node_id == 0): if (os.path.exists(self.trig_start_file)): os.remove(self.trig_start_file) if (os.path.exists(self.trig_stop_file)): os.remove(self.trig_stop_file) else: command = "ssh -n -i %s %s '( rm -f %s %s )'" % ( self.ssh_id_file, self.trigger.node_ip, self.trig_start_file, self.trig_stop_file) print command os.system(command) # Remove initok/initfail files for all ADC nodes for adc in (self.adcboard_list): if (adc.node_id == 0): if (os.path.exists(adc.initok_file_daq)): os.remove(adc.initok_file_daq) if (os.path.exists(adc.initok_file_zsup)): os.remove(adc.initok_file_zsup) if (os.path.exists(adc.initfail_file_daq)): os.remove(adc.initfail_file_daq) if (os.path.exists(adc.initfail_file_zsup)): os.remove(adc.initfail_file_zsup) else: command = "ssh -n -i %s %s '( rm -f %s %s %s %s)'" % ( self.ssh_id_file, adc.node_ip, adc.initok_file_daq, adc.initok_file_zsup, adc.initfail_file_daq, adc.initfail_file_zsup) print command os.system(command) # Remove initok/initfail files on Trigger node if (self.trigger.node_id == 0): if (os.path.exists(self.trigger.initok_file)): os.remove(self.trigger.initok_file) if (os.path.exists(self.trigger.initfail_file)): os.remove(self.trigger.initfail_file) else: command = "ssh -n -i %s %s '( rm -f %s %s)'" % ( self.ssh_id_file, self.trigger.node_ip, self.trigger.initok_file, self.trigger.initfail_file) print command os.system(command) # Stop all receiving nc processes on the merger nodes (clumsy but could not find another way) if (self.merger.node_id == 0): for proc in self.proc_rcv: proc.terminate() else: command = "ssh -i %s %s '( ps -fu %s | grep recv-only | grep -v bash | grep -v grep | awk \"{print \$2}\" | xargs kill )'" % ( self.ssh_id_file, self.merger.node_ip, os.getenv( 'USER', "daq")) print command os.system(command) # Now we can clean up all nc processes for proc in self.proc_snd: if proc.poll() != None: proc.wait() else: print "Run::clean_up - Problem closing sending nc process" for proc in self.proc_rcv: if proc.poll() != None: proc.wait() else: print "Run::clean_up - Problem closing receiving nc process" # Close all receiving/sending nc log files for handler in self.hand_rcv: handler.close() for handler in self.hand_snd: handler.close()