def readConfig(self, global_config_path): """ Read global config and dictionary of all the data_stream configs. """ self.gconfig = GlobalConfig(global_config_path) self.global_config_path = global_config_path # Check configs dir is accessible configs_dir = self.gconfig.get("global.config_dir") if not os.access(configs_dir, os.R_OK): print "Unable to access directory defined in config file:\n\t%s = '%s'.\n\nPlease fix and re-start ." % ("global.config_dir", configs_dir) sys.exit() # Check that we can access various directories and warn users of consequences for check_dir in ("global.base_incoming_dir", "global.top", "global.base_data_dir"): # Check dir and exit if we cannot access it if self.gconfig.get(check_dir).strip() == "": print "WARNING: You have not set the '%s' configuration option in the global configuration. Ensure that the data stream specific configuration files define the full directory paths and do not rely on this global setting." % check_dir elif not os.access(self.gconfig.get(check_dir), os.R_OK): print "WARNING: Unable to access directory defined in config file:\n\t%s = '%s'.\n\nPlease fix and re-start." % (check_dir, self.gconfig.get(check_dir)) sys.exit() # Decide which dataset listing method to use: # 1. ``data_stream_config_dir`` implies get all found in that directory # 2. ``data_stream_list`` implies picking up only those in the directory listed if self.gconfig["global"].has_key("data_stream_config_dir"): self.gconfig["global"]["config_dir"] = self.gconfig["global"]["data_stream_config_dir"] self._populateDatasetConfigsFromConfigDir() print ("Using data_streams in alternative dir: %s" % self.gconfig["global"]["config_dir"]) print "Using data_streams: %s" % self.datasets else: self.datasets = string.split(self.gconfig["global"]["data_stream_list"]) if len(self.datasets) == 0: print "WARNING: No data_stream configuration files found!?! Stopping MiStaMover!" sys.exit() self.dconfigs = {} self._loadDatasetConfigs()
def scanDatasetConfigsForChange(self): """ Reads ``data_stream_config`` dir and reacts to any new config files or any deletions. """ self.gconfig = GlobalConfig(self.global_config_path) self.datasets = string.split(self.gconfig.get("global.data_stream_list")) ds_added = self._loadDatasetConfigs() print "Scanning for new data_stream configs or updated status in existing configs..." # Now start up those that have just been added for ds_name in ds_added: print "Starting procs for data_stream %s" % ds_name self.info("starting procs for data_stream %s" % ds_name) self.startDatasetProcs(ds_name) # And remove any that have been deleted self._removeDeletedDatasets()
class MiStaMoverController(object): """ This is the daemon process that manages a group of other class instances which are run in sub-processes, specifically: * one LoggerServer * one DiskSpaceMonitor for each filesystem (via DiskSpaceMonitorLauncher) * a DatasetTransferController for each data_stream * a DatasetArrivalMonitor for each data_stream which needs it It is the top-level in MiStaMover (apart from the main program MiStaMover.py which basically just sets a few variables and calls this) """ def __init__(self, global_config_path, debug_on=False, oneoff=False): """ Reads global config file """ self.oneoff = oneoff self.global_config_path = global_config_path self.readConfig(global_config_path) if self.oneoff == True: if len(self.datasets) != 1: print "in oneoff mode - only a single data_stream can be transfered at a time" sys.exit() #self.gconfig.dump() if self.gconfig.checkSet("global.debug_on"): self.debug_on = self.gconfig.get("global.debug_on") else: self.debug_on = debug_on # set up some default values if not self.gconfig.checkSet("logging.base_log_dir"): if self.gconfig.checkSet("global.top"): gtop = self.gconfig.get("global.top") self.gconfig.set("logging.base_log_dir", gtop + "/log") if not self.gconfig.checkSet("global.base_data_dir"): if self.gconfig.checkSet("global.top"): gtop = self.gconfig.get("global.top") self.gconfig.set("base.base_data_dir", gtop + "/data") self.checkConfig() self.sub_procs = {} if not os.path.exists(self.gconfig.get("logging.base_log_dir")): print ("log path " + self.gconfig.get("logging.base_log_dir") + " does not exist") sys.exit() self.initLogger() # jah # this seems to be necessary otherwise signals dont seem to be caught signal.signal(signal.SIGINT, self.stopHandler) signal.signal(signal.SIGTERM, self.stopHandler) signal.signal(signal.SIGUSR2, self.stopHandler) def checkConfig(self): if (self.gconfig.checkSet("global.top") == False): raise Exception("global.top must be set") if (self.gconfig.checkSet("global.data_stream_config_dir") == False and self.gconfig.checkSet("global.config_dir") == False): raise Exception("either global.data_stream_config_dir OR global.config_dir must be set") if (self.gconfig.checkSet("global.general_poll_interval") == False): raise Exception("global.general_poll_interval must be set") if (self.gconfig.checkSet("logging.base_log_dir") == False): raise Exception("logging.base_log_dir must be set") if (self.gconfig.checkSet("logging.log_level") == False): raise Exception("logging.log_level must be set") if (self.gconfig.checkSet("logging.port") == False): raise Exception("logging.port must be set") if (self.gconfig.checkSet("email.threshold") == False): raise Exception("email.threshold must be set") if (self.gconfig.checkSet("email.from") == False): raise Exception("email.from must be set") if (self.gconfig.checkSet("email.recipient") == False): raise Exception("email.recipient must be set") if (self.gconfig.checkSet("email.subject") == False): raise Exception("email.subject must be set") if (self.gconfig.checkSet("email.smarthost") == False): raise Exception("email.smart must be set") if (self.gconfig.checkSet("disk_space_monitor.poll_interval") == False): raise Exception("disk_space_monitor.poll_interval must be set") if (self.gconfig.checkSet("disk_space_monitor.base_priority") == False): raise Exception("disk_space_monitor.base_priority must be set") def dumpConfig(self): self.gconfig.dump() for d in self.dconfigs.values(): d.dump() def main(self): """ Main loop. Start the daemons, then wait until we are told to stop. It checks the data_stream config directory for new data_streams if that method is used. """ self.startAll() self.stopRequested = False for signo in (signal.SIGTERM, signal.SIGINT): signal.signal(signo, self.stopHandler) while not self.stopRequested: # any signal will cause sleep to terminate early # after 30 seconds test for changes to configs dir time.sleep(30) self.scanDatasetConfigsForChange() self.stopAll() def stopHandler(self, signo, frame): """ A signal handler that just records that we were signalled. Doesn't bother to see what signal we were called with, as should only be installed as a handler for appropriate signals. """ if self.logger: self.info("Received signal %s" % signo) if signo != signal.SIGUSR2: self.stopRequested = True self.stopAll() else: print "sigusr1 was handled" totalDataSetsRunning = 0 for k in self.sub_procs: print k v = self.sub_procs[k] # d is a daemonCtl instance d = v['sender'] time.sleep(0.1) if d.isRunning(): totalDataSetsRunning += 1 if totalDataSetsRunning == 0: print "no data_streams seem to be running" self.stopRequested = True self.stopAll() def readConfig(self, global_config_path): """ Read global config and dictionary of all the data_stream configs. """ self.gconfig = GlobalConfig(global_config_path) self.global_config_path = global_config_path # Check configs dir is accessible configs_dir = self.gconfig.get("global.config_dir") if not os.access(configs_dir, os.R_OK): print "Unable to access directory defined in config file:\n\t%s = '%s'.\n\nPlease fix and re-start ." % ("global.config_dir", configs_dir) sys.exit() # Check that we can access various directories and warn users of consequences for check_dir in ("global.base_incoming_dir", "global.top", "global.base_data_dir"): # Check dir and exit if we cannot access it if self.gconfig.get(check_dir).strip() == "": print "WARNING: You have not set the '%s' configuration option in the global configuration. Ensure that the data stream specific configuration files define the full directory paths and do not rely on this global setting." % check_dir elif not os.access(self.gconfig.get(check_dir), os.R_OK): print "WARNING: Unable to access directory defined in config file:\n\t%s = '%s'.\n\nPlease fix and re-start." % (check_dir, self.gconfig.get(check_dir)) sys.exit() # Decide which dataset listing method to use: # 1. ``data_stream_config_dir`` implies get all found in that directory # 2. ``data_stream_list`` implies picking up only those in the directory listed if self.gconfig["global"].has_key("data_stream_config_dir"): self.gconfig["global"]["config_dir"] = self.gconfig["global"]["data_stream_config_dir"] self._populateDatasetConfigsFromConfigDir() print ("Using data_streams in alternative dir: %s" % self.gconfig["global"]["config_dir"]) print "Using data_streams: %s" % self.datasets else: self.datasets = string.split(self.gconfig["global"]["data_stream_list"]) if len(self.datasets) == 0: print "WARNING: No data_stream configuration files found!?! Stopping MiStaMover!" sys.exit() self.dconfigs = {} self._loadDatasetConfigs() def _loadDatasetConfigs(self): """ Goes through and loads up data_stream configs if not already there. Returns a list of those data_streams that have just been added. """ ds_added = [] for ds_name in self.datasets: data_stream_path = self.gconfig.get("global.config_dir") # check that the config files have 400 permissions ds_filepath = data_stream_path + "/ds_" + ds_name + ".ini" try: st = os.stat(ds_filepath).st_mode except: print "" wrn = "The global configuration 'data_stream_list' setting expects to find a configuration for the '" + ds_name + "' data stream called: '" + ds_filepath + "'. This file does not exist. Please create it manually or use the 'bin/create_config.py' wizard." print "WARNING", wrn continue rv = stat.S_IMODE(st) if rv == stat.S_IRUSR: if ds_name not in self.dconfigs.keys(): if self.oneoff == False: self.dconfigs[ds_name] = DatasetConfig(ds_name, self.gconfig) else: self.dconfigs[ds_name] = (DatasetConfig(ds_name, self.gconfig, self.global_config_path)) ds_added.append(ds_name) # Now they should all have been loaded. When re-trying, try all those # that are STOPPED as they might have been re-started ds_status = self.dconfigs[ds_name].get("data_stream.status") if ds_status == "<stopped>": print "Re-scanning previously stopped data_stream just in case re-started: %s" % ds_name self.dconfigs[ds_name] = \ DatasetConfig(ds_name, self.gconfig) new_ds_status = self.dconfigs[ds_name].get("data_stream.status") if new_ds_status == "<running>": ds_added.append(ds_name) else: print ds_filepath + " must have permissions of 400" if hasattr(self, 'info'): self.info("%s must have permissions of 400" % ds_filepath) # remove the offending item self.datasets.remove(ds_name) return ds_added def _removeDeletedDatasets(self): """ Remove any (and try stopping the sub-process) data_streams that have disappeared from the configuration. """ for ds_name in self.dconfigs.keys(): if ds_name not in self.datasets: (self.info("STOPPING data_stream because removed from data_stream list: %s" % ds_name)) self.stopDatasetProcs(ds_name) del self.dconfigs[ds_name] def _populateDatasetConfigsFromConfigDir(self): """ Lists all files in ``data_stream_config_dir`` to generate a list of data_streams to use. Stores them in ``self.data_streams``. """ if self.gconfig["global"].has_key("data_stream_config_dir"): self.gconfig["global"]["config_dir"] = self.gconfig["global"]["data_stream_config_dir"] dataset_config_dir = self.gconfig.get("global.config_dir") print ("Scanning data_stream configs dir for new data streams: %s" % dataset_config_dir) dataset_config_files = ([conf_file for conf_file in os.listdir(dataset_config_dir) if conf_file[0] != "."]) self.datasets = ([i.replace("dataset_", "").replace(".ini", "") for i in dataset_config_files]) else: pass def scanDatasetConfigsForChange(self): """ Reads ``data_stream_config`` dir and reacts to any new config files or any deletions. """ self.gconfig = GlobalConfig(self.global_config_path) self.datasets = string.split(self.gconfig.get("global.data_stream_list")) ds_added = self._loadDatasetConfigs() print "Scanning for new data_stream configs or updated status in existing configs..." # Now start up those that have just been added for ds_name in ds_added: print "Starting procs for data_stream %s" % ds_name self.info("starting procs for data_stream %s" % ds_name) self.startDatasetProcs(ds_name) # And remove any that have been deleted self._removeDeletedDatasets() def startAll(self): """ Starts up sub-processes: log server, disk space monitor, and data_stream specific processes """ self.startLogServer() self.dsml = DiskSpaceMonitorLauncher.DiskSpaceMonitorLauncher( self.gconfig, self.dconfigs, logger = self.logger) self.dsml.launch() self.info("started log server") for ds_name in self.datasets: self.info("starting procs for data_stream %s" % ds_name) self.startDatasetProcs(ds_name) def stopAll(self): """ Stop the same things that startAll() starts """ for ds_name in self.datasets: self.info("signalling procs for data_stream %s to stop" % ds_name) self.stopDatasetProcs(ds_name) for ds_name in self.datasets: self.info("waiting for procs for data_stream %s to stop" % ds_name) self.waitForDatasetProcs(ds_name) # stop disk space monitor try: self.dsml.kill() except Exception: pass self.stopLogServer() for k in self.sub_procs: v = self.sub_procs[k] for j in v: q = v[j] try: os.kill(q.pid, signal.SIGUSR1) except: pass def waitForDatasetProcs(self, ds_name): """ Wait for processes associated with a named data_stream to stop """ daemons = self.getDatasetProcs(ds_name) for d in daemons: if d.isRunning(): status = d.getStatus(wait = True) self.info("process '%s' exited with status %s" % \ (d.description, status)) def initLogger(self): """ initialise the logger client object for logging from this module. NB this is completely separate from the server side, which is also started by MiStaMoverController """ self.logger = LoggerClient.LoggerClient( self.gconfig, tag = "mistamover_ctl", name = "top-level mistamover controller", debug_on = self.debug_on) self.logger.exportMethods(self) def startLogServer(self): """ Start the logger server process """ self.log_server_proc = Daemon.DaemonCtl(self.runLogger, args = [self.gconfig], description = "mistamover_log_server") def stopLogServer(self): try: self.log_server_proc.shutdown() except Exception, ex: str(ex)