Exemplo n.º 1
0
    def readConfig(self, global_config_path):
        """
        Read global config and dictionary of all the data_stream configs.
        """
        self.gconfig = GlobalConfig(global_config_path)
        self.global_config_path = global_config_path
        # Check configs dir is accessible
        configs_dir = self.gconfig.get("global.config_dir")
        if not os.access(configs_dir, os.R_OK):
            print "Unable to access directory defined in config file:\n\t%s = '%s'.\n\nPlease fix and re-start ." % ("global.config_dir", configs_dir)
            sys.exit()
        # Check that we can access various directories and warn users of consequences
        for check_dir in ("global.base_incoming_dir",
                          "global.top",
                          "global.base_data_dir"):
            # Check dir and exit if we cannot access it
            if self.gconfig.get(check_dir).strip() == "":
                print "WARNING: You have not set the '%s' configuration option in the global configuration. Ensure that the data stream specific configuration files define the full directory paths and do not rely on this global setting." % check_dir
            elif not os.access(self.gconfig.get(check_dir), os.R_OK):
                print "WARNING: Unable to access directory defined in config file:\n\t%s = '%s'.\n\nPlease fix and re-start." % (check_dir, self.gconfig.get(check_dir))
                sys.exit()


        # Decide which dataset listing method to use:
        #  1. ``data_stream_config_dir`` implies get all found in that directory
        #  2. ``data_stream_list`` implies picking up only those in the directory listed

        if self.gconfig["global"].has_key("data_stream_config_dir"):
            self.gconfig["global"]["config_dir"] = self.gconfig["global"]["data_stream_config_dir"]
            self._populateDatasetConfigsFromConfigDir()
            print ("Using data_streams in alternative dir: %s" % 
                self.gconfig["global"]["config_dir"])
            print "Using data_streams: %s" % self.datasets
        else:
            self.datasets = string.split(self.gconfig["global"]["data_stream_list"])

        if len(self.datasets) == 0:
            print "WARNING: No data_stream configuration files found!?! Stopping MiStaMover!"
            sys.exit()

        self.dconfigs = {}
        self._loadDatasetConfigs()
Exemplo n.º 2
0
    def scanDatasetConfigsForChange(self):
        """
        Reads ``data_stream_config`` dir and reacts to any new config files or 
        any deletions.
        """
        self.gconfig = GlobalConfig(self.global_config_path)
        self.datasets = string.split(self.gconfig.get("global.data_stream_list"))

        ds_added = self._loadDatasetConfigs()        

        print "Scanning for new data_stream configs or updated status in existing configs..."

        # Now start up those that have just been added
        for ds_name in ds_added:
            print "Starting procs for data_stream %s" % ds_name
            self.info("starting procs for data_stream %s" % ds_name)
            self.startDatasetProcs(ds_name) 
           
        # And remove any that have been deleted 
        self._removeDeletedDatasets()
Exemplo n.º 3
0
class MiStaMoverController(object):
    """
    This is the daemon process that manages a group of other class 
    instances which are run in sub-processes, specifically:

      * one LoggerServer

      * one DiskSpaceMonitor for each filesystem 
           (via DiskSpaceMonitorLauncher)

      * a DatasetTransferController for each data_stream

      * a DatasetArrivalMonitor for each data_stream which needs it

    It is the top-level in MiStaMover (apart from the main program MiStaMover.py
    which basically just sets a few variables and calls this)
    """  

    def __init__(self, global_config_path, debug_on=False, oneoff=False):
        """
        Reads global config file
        """
        self.oneoff = oneoff
        self.global_config_path = global_config_path
        self.readConfig(global_config_path)

        if self.oneoff == True:
            if len(self.datasets) != 1:
                print "in oneoff mode - only a single data_stream can be transfered at a time"
                sys.exit()

        #self.gconfig.dump()
        if self.gconfig.checkSet("global.debug_on"):
            self.debug_on = self.gconfig.get("global.debug_on")
        else:
            self.debug_on = debug_on

        # set up some default values
        if not self.gconfig.checkSet("logging.base_log_dir"):
            if self.gconfig.checkSet("global.top"):
                gtop = self.gconfig.get("global.top")
                self.gconfig.set("logging.base_log_dir", gtop + "/log")

        if not self.gconfig.checkSet("global.base_data_dir"):
            if self.gconfig.checkSet("global.top"):
                gtop = self.gconfig.get("global.top")
                self.gconfig.set("base.base_data_dir", gtop + "/data")

        self.checkConfig()

        self.sub_procs = {}

        if not os.path.exists(self.gconfig.get("logging.base_log_dir")):
            print ("log path " + self.gconfig.get("logging.base_log_dir") + 
                " does not exist")
            sys.exit()

        self.initLogger()

        # jah
        # this seems to be necessary otherwise signals dont seem to be caught
        signal.signal(signal.SIGINT, self.stopHandler)
        signal.signal(signal.SIGTERM, self.stopHandler)
        signal.signal(signal.SIGUSR2, self.stopHandler)        

    def checkConfig(self):
      if (self.gconfig.checkSet("global.top") == False):
          raise Exception("global.top must be set")
      if (self.gconfig.checkSet("global.data_stream_config_dir") == False and
          self.gconfig.checkSet("global.config_dir") == False):
          raise Exception("either global.data_stream_config_dir OR global.config_dir must be set")
      if (self.gconfig.checkSet("global.general_poll_interval") == False):
          raise Exception("global.general_poll_interval must be set")
      if (self.gconfig.checkSet("logging.base_log_dir") == False):
          raise Exception("logging.base_log_dir must be set")
      if (self.gconfig.checkSet("logging.log_level") == False):
          raise Exception("logging.log_level must be set")
      if (self.gconfig.checkSet("logging.port") == False):
          raise Exception("logging.port must be set")
      if (self.gconfig.checkSet("email.threshold") == False):
          raise Exception("email.threshold must be set")
      if (self.gconfig.checkSet("email.from") == False):
          raise Exception("email.from must be set")
      if (self.gconfig.checkSet("email.recipient") == False):
          raise Exception("email.recipient must be set")
      if (self.gconfig.checkSet("email.subject") == False):
          raise Exception("email.subject must be set")
      if (self.gconfig.checkSet("email.smarthost") == False):
          raise Exception("email.smart must be set")
      if (self.gconfig.checkSet("disk_space_monitor.poll_interval") == False):
          raise Exception("disk_space_monitor.poll_interval must be set")
      if (self.gconfig.checkSet("disk_space_monitor.base_priority") == False):
          raise Exception("disk_space_monitor.base_priority must be set")

    def dumpConfig(self):
        self.gconfig.dump()
        for d in self.dconfigs.values():
            d.dump()

    def main(self):
        """
        Main loop.
        Start the daemons, then wait until we are told to stop.
        It checks the data_stream config directory for new data_streams if that
        method is used.
        """

        self.startAll()
        self.stopRequested = False
        for signo in (signal.SIGTERM, signal.SIGINT):
            signal.signal(signo, self.stopHandler)
        
        while not self.stopRequested:
            # any signal will cause sleep to terminate early
            # after 30 seconds test for changes to configs dir
            time.sleep(30)
            self.scanDatasetConfigsForChange()
    
        self.stopAll()
                


    def stopHandler(self, signo, frame):
        """
        A signal handler that just records that we were signalled.
        Doesn't bother to see what signal we were called with, as
        should only be installed as a handler for appropriate signals.
        """
        if self.logger:
            self.info("Received signal %s" % signo)
        if signo != signal.SIGUSR2:
            self.stopRequested = True
            self.stopAll()
        else:
            print "sigusr1 was handled"
            totalDataSetsRunning = 0
            for k in self.sub_procs:
                print k
                v = self.sub_procs[k]
                # d is a daemonCtl instance
                d = v['sender']
                time.sleep(0.1)
                if d.isRunning():
                    totalDataSetsRunning += 1
            if totalDataSetsRunning == 0:
                print "no data_streams seem to be running"
                self.stopRequested = True
                self.stopAll()

    def readConfig(self, global_config_path):
        """
        Read global config and dictionary of all the data_stream configs.
        """
        self.gconfig = GlobalConfig(global_config_path)
        self.global_config_path = global_config_path
        # Check configs dir is accessible
        configs_dir = self.gconfig.get("global.config_dir")
        if not os.access(configs_dir, os.R_OK):
            print "Unable to access directory defined in config file:\n\t%s = '%s'.\n\nPlease fix and re-start ." % ("global.config_dir", configs_dir)
            sys.exit()
        # Check that we can access various directories and warn users of consequences
        for check_dir in ("global.base_incoming_dir",
                          "global.top",
                          "global.base_data_dir"):
            # Check dir and exit if we cannot access it
            if self.gconfig.get(check_dir).strip() == "":
                print "WARNING: You have not set the '%s' configuration option in the global configuration. Ensure that the data stream specific configuration files define the full directory paths and do not rely on this global setting." % check_dir
            elif not os.access(self.gconfig.get(check_dir), os.R_OK):
                print "WARNING: Unable to access directory defined in config file:\n\t%s = '%s'.\n\nPlease fix and re-start." % (check_dir, self.gconfig.get(check_dir))
                sys.exit()


        # Decide which dataset listing method to use:
        #  1. ``data_stream_config_dir`` implies get all found in that directory
        #  2. ``data_stream_list`` implies picking up only those in the directory listed

        if self.gconfig["global"].has_key("data_stream_config_dir"):
            self.gconfig["global"]["config_dir"] = self.gconfig["global"]["data_stream_config_dir"]
            self._populateDatasetConfigsFromConfigDir()
            print ("Using data_streams in alternative dir: %s" % 
                self.gconfig["global"]["config_dir"])
            print "Using data_streams: %s" % self.datasets
        else:
            self.datasets = string.split(self.gconfig["global"]["data_stream_list"])

        if len(self.datasets) == 0:
            print "WARNING: No data_stream configuration files found!?! Stopping MiStaMover!"
            sys.exit()

        self.dconfigs = {}
        self._loadDatasetConfigs()

    def _loadDatasetConfigs(self):
        """
        Goes through and loads up data_stream configs if not already there. 
        Returns a list of those data_streams that have just been added.
        """

        ds_added = []  

        for ds_name in self.datasets:
            data_stream_path = self.gconfig.get("global.config_dir")
            # check that the config files have 400 permissions
            ds_filepath = data_stream_path + "/ds_" + ds_name + ".ini"
            try:
                st = os.stat(ds_filepath).st_mode
            except:
                print ""
                wrn = "The global configuration 'data_stream_list' setting expects to find a configuration for the '" + ds_name + "' data stream called: '" + ds_filepath + "'. This file does not exist. Please create it manually or use the 'bin/create_config.py' wizard."
                print "WARNING", wrn
                continue
            rv = stat.S_IMODE(st)
            if rv == stat.S_IRUSR:
                if ds_name not in self.dconfigs.keys():
                    if self.oneoff == False:
                        self.dconfigs[ds_name] = DatasetConfig(ds_name, self.gconfig)
                    else:
                        self.dconfigs[ds_name] = (DatasetConfig(ds_name, self.gconfig,
                            self.global_config_path))
                    ds_added.append(ds_name)

                # Now they should all have been loaded. When re-trying, try all those
                # that are STOPPED as they might have been re-started
                ds_status = self.dconfigs[ds_name].get("data_stream.status")

                if ds_status == "<stopped>":
                    print "Re-scanning previously stopped data_stream just in case re-started: %s" % ds_name

                    self.dconfigs[ds_name] = \
                        DatasetConfig(ds_name, self.gconfig)

                    new_ds_status = self.dconfigs[ds_name].get("data_stream.status")
                    if new_ds_status == "<running>":
                        ds_added.append(ds_name)
            else:
                print ds_filepath + " must have permissions of 400"
                if hasattr(self, 'info'):
                    self.info("%s must have permissions of 400" % ds_filepath)
                # remove the offending item
                self.datasets.remove(ds_name)
        return ds_added


    def _removeDeletedDatasets(self):
        """
        Remove any (and try stopping the sub-process) data_streams that have 
        disappeared from the configuration.
        """
        for ds_name in self.dconfigs.keys():

            if ds_name not in self.datasets:
                (self.info("STOPPING data_stream because removed from data_stream list: %s" %
                    ds_name))
                self.stopDatasetProcs(ds_name) 
                del self.dconfigs[ds_name]


    def _populateDatasetConfigsFromConfigDir(self):
        """
        Lists all files in ``data_stream_config_dir`` to generate a list of 
        data_streams to use. Stores them in ``self.data_streams``.
        """

        if self.gconfig["global"].has_key("data_stream_config_dir"):
            self.gconfig["global"]["config_dir"] = self.gconfig["global"]["data_stream_config_dir"]
            dataset_config_dir = self.gconfig.get("global.config_dir")

            print ("Scanning data_stream configs dir for new data streams: %s" % 
                dataset_config_dir)
            dataset_config_files = ([conf_file for conf_file in 
                os.listdir(dataset_config_dir) if conf_file[0] != "."])
            self.datasets = ([i.replace("dataset_", "").replace(".ini", "") 
                for i in dataset_config_files])
        else:
            pass


    def scanDatasetConfigsForChange(self):
        """
        Reads ``data_stream_config`` dir and reacts to any new config files or 
        any deletions.
        """
        self.gconfig = GlobalConfig(self.global_config_path)
        self.datasets = string.split(self.gconfig.get("global.data_stream_list"))

        ds_added = self._loadDatasetConfigs()        

        print "Scanning for new data_stream configs or updated status in existing configs..."

        # Now start up those that have just been added
        for ds_name in ds_added:
            print "Starting procs for data_stream %s" % ds_name
            self.info("starting procs for data_stream %s" % ds_name)
            self.startDatasetProcs(ds_name) 
           
        # And remove any that have been deleted 
        self._removeDeletedDatasets()

        
    def startAll(self):
        """
        Starts up sub-processes: log server, disk space monitor, 
          and data_stream specific processes
        """
        self.startLogServer()

        self.dsml = DiskSpaceMonitorLauncher.DiskSpaceMonitorLauncher(
            self.gconfig, self.dconfigs, logger = self.logger)
        self.dsml.launch()

        self.info("started log server")
        for ds_name in self.datasets:
            self.info("starting procs for data_stream %s" % ds_name)
            self.startDatasetProcs(ds_name)


    def stopAll(self):
        """
        Stop the same things that startAll() starts
        """
        for ds_name in self.datasets:
            self.info("signalling procs for data_stream %s to stop" %
                             ds_name)
            self.stopDatasetProcs(ds_name)
        for ds_name in self.datasets:
            self.info("waiting for procs for data_stream %s to stop" %
                             ds_name)
            self.waitForDatasetProcs(ds_name)

        # stop disk space monitor
        try:
            self.dsml.kill()
        except Exception:
            pass 
        self.stopLogServer()

        for k in self.sub_procs:
            v = self.sub_procs[k]
            for j in v:
                q = v[j]
                try:
                    os.kill(q.pid, signal.SIGUSR1)
                except:
                    pass
            
    def waitForDatasetProcs(self, ds_name):
        """
        Wait for processes associated with a named data_stream to stop
        """
        daemons = self.getDatasetProcs(ds_name)
        for d in daemons:
            if d.isRunning():
                status = d.getStatus(wait = True)
                self.info("process '%s' exited with status %s" % \
                                 (d.description, status))


    def initLogger(self):
        """
        initialise the logger client object for logging from this module.
        NB this is completely separate from the server side, which is also 
        started by MiStaMoverController
        """
        self.logger = LoggerClient.LoggerClient(
            self.gconfig,
            tag = "mistamover_ctl",
            name = "top-level mistamover controller",
            debug_on = self.debug_on)
        self.logger.exportMethods(self)

        
    def startLogServer(self):
        """
        Start the logger server process
        """
        self.log_server_proc = Daemon.DaemonCtl(self.runLogger,
                                                args = [self.gconfig],
                                                description = "mistamover_log_server")
        
    def stopLogServer(self):
        try:
            self.log_server_proc.shutdown()
        except Exception, ex:
            str(ex)