def readConfig(self, global_config_path):
"""
Read global config and dictionary of all the data_stream configs.
"""
self.gconfig = GlobalConfig(global_config_path)
self.global_config_path = global_config_path
# Check configs dir is accessible
configs_dir = self.gconfig.get("global.config_dir")
if not os.access(configs_dir, os.R_OK):
print "Unable to access directory defined in config file:\n\t%s = '%s'.\n\nPlease fix and re-start ." % ("global.config_dir", configs_dir)
sys.exit()
# Check that we can access various directories and warn users of consequences
for check_dir in ("global.base_incoming_dir",
"global.top",
"global.base_data_dir"):
# Check dir and exit if we cannot access it
if self.gconfig.get(check_dir).strip() == "":
print "WARNING: You have not set the '%s' configuration option in the global configuration. Ensure that the data stream specific configuration files define the full directory paths and do not rely on this global setting." % check_dir
elif not os.access(self.gconfig.get(check_dir), os.R_OK):
print "WARNING: Unable to access directory defined in config file:\n\t%s = '%s'.\n\nPlease fix and re-start." % (check_dir, self.gconfig.get(check_dir))
sys.exit()
# Decide which dataset listing method to use:
# 1. ``data_stream_config_dir`` implies get all found in that directory
# 2. ``data_stream_list`` implies picking up only those in the directory listed
if self.gconfig["global"].has_key("data_stream_config_dir"):
self.gconfig["global"]["config_dir"] = self.gconfig["global"]["data_stream_config_dir"]
self._populateDatasetConfigsFromConfigDir()
print ("Using data_streams in alternative dir: %s" %
self.gconfig["global"]["config_dir"])
print "Using data_streams: %s" % self.datasets
else:
self.datasets = string.split(self.gconfig["global"]["data_stream_list"])
if len(self.datasets) == 0:
print "WARNING: No data_stream configuration files found!?! Stopping MiStaMover!"
sys.exit()
self.dconfigs = {}
self._loadDatasetConfigs()
def scanDatasetConfigsForChange(self):
"""
Reads ``data_stream_config`` dir and reacts to any new config files or
any deletions.
"""
self.gconfig = GlobalConfig(self.global_config_path)
self.datasets = string.split(self.gconfig.get("global.data_stream_list"))
ds_added = self._loadDatasetConfigs()
print "Scanning for new data_stream configs or updated status in existing configs..."
# Now start up those that have just been added
for ds_name in ds_added:
print "Starting procs for data_stream %s" % ds_name
self.info("starting procs for data_stream %s" % ds_name)
self.startDatasetProcs(ds_name)
# And remove any that have been deleted
self._removeDeletedDatasets()
class MiStaMoverController(object):
"""
This is the daemon process that manages a group of other class
instances which are run in sub-processes, specifically:
* one LoggerServer
* one DiskSpaceMonitor for each filesystem
(via DiskSpaceMonitorLauncher)
* a DatasetTransferController for each data_stream
* a DatasetArrivalMonitor for each data_stream which needs it
It is the top-level in MiStaMover (apart from the main program MiStaMover.py
which basically just sets a few variables and calls this)
"""
def __init__(self, global_config_path, debug_on=False, oneoff=False):
"""
Reads global config file
"""
self.oneoff = oneoff
self.global_config_path = global_config_path
self.readConfig(global_config_path)
if self.oneoff == True:
if len(self.datasets) != 1:
print "in oneoff mode - only a single data_stream can be transfered at a time"
sys.exit()
#self.gconfig.dump()
if self.gconfig.checkSet("global.debug_on"):
self.debug_on = self.gconfig.get("global.debug_on")
else:
self.debug_on = debug_on
# set up some default values
if not self.gconfig.checkSet("logging.base_log_dir"):
if self.gconfig.checkSet("global.top"):
gtop = self.gconfig.get("global.top")
self.gconfig.set("logging.base_log_dir", gtop + "/log")
if not self.gconfig.checkSet("global.base_data_dir"):
if self.gconfig.checkSet("global.top"):
gtop = self.gconfig.get("global.top")
self.gconfig.set("base.base_data_dir", gtop + "/data")
self.checkConfig()
self.sub_procs = {}
if not os.path.exists(self.gconfig.get("logging.base_log_dir")):
print ("log path " + self.gconfig.get("logging.base_log_dir") +
" does not exist")
sys.exit()
self.initLogger()
# jah
# this seems to be necessary otherwise signals dont seem to be caught
signal.signal(signal.SIGINT, self.stopHandler)
signal.signal(signal.SIGTERM, self.stopHandler)
signal.signal(signal.SIGUSR2, self.stopHandler)
def checkConfig(self):
if (self.gconfig.checkSet("global.top") == False):
raise Exception("global.top must be set")
if (self.gconfig.checkSet("global.data_stream_config_dir") == False and
self.gconfig.checkSet("global.config_dir") == False):
raise Exception("either global.data_stream_config_dir OR global.config_dir must be set")
if (self.gconfig.checkSet("global.general_poll_interval") == False):
raise Exception("global.general_poll_interval must be set")
if (self.gconfig.checkSet("logging.base_log_dir") == False):
raise Exception("logging.base_log_dir must be set")
if (self.gconfig.checkSet("logging.log_level") == False):
raise Exception("logging.log_level must be set")
if (self.gconfig.checkSet("logging.port") == False):
raise Exception("logging.port must be set")
if (self.gconfig.checkSet("email.threshold") == False):
raise Exception("email.threshold must be set")
if (self.gconfig.checkSet("email.from") == False):
raise Exception("email.from must be set")
if (self.gconfig.checkSet("email.recipient") == False):
raise Exception("email.recipient must be set")
if (self.gconfig.checkSet("email.subject") == False):
raise Exception("email.subject must be set")
if (self.gconfig.checkSet("email.smarthost") == False):
raise Exception("email.smart must be set")
if (self.gconfig.checkSet("disk_space_monitor.poll_interval") == False):
raise Exception("disk_space_monitor.poll_interval must be set")
if (self.gconfig.checkSet("disk_space_monitor.base_priority") == False):
raise Exception("disk_space_monitor.base_priority must be set")
def dumpConfig(self):
self.gconfig.dump()
for d in self.dconfigs.values():
d.dump()
def main(self):
"""
Main loop.
Start the daemons, then wait until we are told to stop.
It checks the data_stream config directory for new data_streams if that
method is used.
"""
self.startAll()
self.stopRequested = False
for signo in (signal.SIGTERM, signal.SIGINT):
signal.signal(signo, self.stopHandler)
while not self.stopRequested:
# any signal will cause sleep to terminate early
# after 30 seconds test for changes to configs dir
time.sleep(30)
self.scanDatasetConfigsForChange()
self.stopAll()
def stopHandler(self, signo, frame):
"""
A signal handler that just records that we were signalled.
Doesn't bother to see what signal we were called with, as
should only be installed as a handler for appropriate signals.
"""
if self.logger:
self.info("Received signal %s" % signo)
if signo != signal.SIGUSR2:
self.stopRequested = True
self.stopAll()
else:
print "sigusr1 was handled"
totalDataSetsRunning = 0
for k in self.sub_procs:
print k
v = self.sub_procs[k]
# d is a daemonCtl instance
d = v['sender']
time.sleep(0.1)
if d.isRunning():
totalDataSetsRunning += 1
if totalDataSetsRunning == 0:
print "no data_streams seem to be running"
self.stopRequested = True
self.stopAll()
def readConfig(self, global_config_path):
"""
Read global config and dictionary of all the data_stream configs.
"""
self.gconfig = GlobalConfig(global_config_path)
self.global_config_path = global_config_path
# Check configs dir is accessible
configs_dir = self.gconfig.get("global.config_dir")
if not os.access(configs_dir, os.R_OK):
print "Unable to access directory defined in config file:\n\t%s = '%s'.\n\nPlease fix and re-start ." % ("global.config_dir", configs_dir)
sys.exit()
# Check that we can access various directories and warn users of consequences
for check_dir in ("global.base_incoming_dir",
"global.top",
"global.base_data_dir"):
# Check dir and exit if we cannot access it
if self.gconfig.get(check_dir).strip() == "":
print "WARNING: You have not set the '%s' configuration option in the global configuration. Ensure that the data stream specific configuration files define the full directory paths and do not rely on this global setting." % check_dir
elif not os.access(self.gconfig.get(check_dir), os.R_OK):
print "WARNING: Unable to access directory defined in config file:\n\t%s = '%s'.\n\nPlease fix and re-start." % (check_dir, self.gconfig.get(check_dir))
sys.exit()
# Decide which dataset listing method to use:
# 1. ``data_stream_config_dir`` implies get all found in that directory
# 2. ``data_stream_list`` implies picking up only those in the directory listed
if self.gconfig["global"].has_key("data_stream_config_dir"):
self.gconfig["global"]["config_dir"] = self.gconfig["global"]["data_stream_config_dir"]
self._populateDatasetConfigsFromConfigDir()
print ("Using data_streams in alternative dir: %s" %
self.gconfig["global"]["config_dir"])
print "Using data_streams: %s" % self.datasets
else:
self.datasets = string.split(self.gconfig["global"]["data_stream_list"])
if len(self.datasets) == 0:
print "WARNING: No data_stream configuration files found!?! Stopping MiStaMover!"
sys.exit()
self.dconfigs = {}
self._loadDatasetConfigs()
def _loadDatasetConfigs(self):
"""
Goes through and loads up data_stream configs if not already there.
Returns a list of those data_streams that have just been added.
"""
ds_added = []
for ds_name in self.datasets:
data_stream_path = self.gconfig.get("global.config_dir")
# check that the config files have 400 permissions
ds_filepath = data_stream_path + "/ds_" + ds_name + ".ini"
try:
st = os.stat(ds_filepath).st_mode
except:
print ""
wrn = "The global configuration 'data_stream_list' setting expects to find a configuration for the '" + ds_name + "' data stream called: '" + ds_filepath + "'. This file does not exist. Please create it manually or use the 'bin/create_config.py' wizard."
print "WARNING", wrn
continue
rv = stat.S_IMODE(st)
if rv == stat.S_IRUSR:
if ds_name not in self.dconfigs.keys():
if self.oneoff == False:
self.dconfigs[ds_name] = DatasetConfig(ds_name, self.gconfig)
else:
self.dconfigs[ds_name] = (DatasetConfig(ds_name, self.gconfig,
self.global_config_path))
ds_added.append(ds_name)
# Now they should all have been loaded. When re-trying, try all those
# that are STOPPED as they might have been re-started
ds_status = self.dconfigs[ds_name].get("data_stream.status")
if ds_status == "<stopped>":
print "Re-scanning previously stopped data_stream just in case re-started: %s" % ds_name
self.dconfigs[ds_name] = \
DatasetConfig(ds_name, self.gconfig)
new_ds_status = self.dconfigs[ds_name].get("data_stream.status")
if new_ds_status == "<running>":
ds_added.append(ds_name)
else:
print ds_filepath + " must have permissions of 400"
if hasattr(self, 'info'):
self.info("%s must have permissions of 400" % ds_filepath)
# remove the offending item
self.datasets.remove(ds_name)
return ds_added
def _removeDeletedDatasets(self):
"""
Remove any (and try stopping the sub-process) data_streams that have
disappeared from the configuration.
"""
for ds_name in self.dconfigs.keys():
if ds_name not in self.datasets:
(self.info("STOPPING data_stream because removed from data_stream list: %s" %
ds_name))
self.stopDatasetProcs(ds_name)
del self.dconfigs[ds_name]
def _populateDatasetConfigsFromConfigDir(self):
"""
Lists all files in ``data_stream_config_dir`` to generate a list of
data_streams to use. Stores them in ``self.data_streams``.
"""
if self.gconfig["global"].has_key("data_stream_config_dir"):
self.gconfig["global"]["config_dir"] = self.gconfig["global"]["data_stream_config_dir"]
dataset_config_dir = self.gconfig.get("global.config_dir")
print ("Scanning data_stream configs dir for new data streams: %s" %
dataset_config_dir)
dataset_config_files = ([conf_file for conf_file in
os.listdir(dataset_config_dir) if conf_file[0] != "."])
self.datasets = ([i.replace("dataset_", "").replace(".ini", "")
for i in dataset_config_files])
else:
pass
def scanDatasetConfigsForChange(self):
"""
Reads ``data_stream_config`` dir and reacts to any new config files or
any deletions.
"""
self.gconfig = GlobalConfig(self.global_config_path)
self.datasets = string.split(self.gconfig.get("global.data_stream_list"))
ds_added = self._loadDatasetConfigs()
print "Scanning for new data_stream configs or updated status in existing configs..."
# Now start up those that have just been added
for ds_name in ds_added:
print "Starting procs for data_stream %s" % ds_name
self.info("starting procs for data_stream %s" % ds_name)
self.startDatasetProcs(ds_name)
# And remove any that have been deleted
self._removeDeletedDatasets()
def startAll(self):
"""
Starts up sub-processes: log server, disk space monitor,
and data_stream specific processes
"""
self.startLogServer()
self.dsml = DiskSpaceMonitorLauncher.DiskSpaceMonitorLauncher(
self.gconfig, self.dconfigs, logger = self.logger)
self.dsml.launch()
self.info("started log server")
for ds_name in self.datasets:
self.info("starting procs for data_stream %s" % ds_name)
self.startDatasetProcs(ds_name)
def stopAll(self):
"""
Stop the same things that startAll() starts
"""
for ds_name in self.datasets:
self.info("signalling procs for data_stream %s to stop" %
ds_name)
self.stopDatasetProcs(ds_name)
for ds_name in self.datasets:
self.info("waiting for procs for data_stream %s to stop" %
ds_name)
self.waitForDatasetProcs(ds_name)
# stop disk space monitor
try:
self.dsml.kill()
except Exception:
pass
self.stopLogServer()
for k in self.sub_procs:
v = self.sub_procs[k]
for j in v:
q = v[j]
try:
os.kill(q.pid, signal.SIGUSR1)
except:
pass
def waitForDatasetProcs(self, ds_name):
"""
Wait for processes associated with a named data_stream to stop
"""
daemons = self.getDatasetProcs(ds_name)
for d in daemons:
if d.isRunning():
status = d.getStatus(wait = True)
self.info("process '%s' exited with status %s" % \
(d.description, status))
def initLogger(self):
"""
initialise the logger client object for logging from this module.
NB this is completely separate from the server side, which is also
started by MiStaMoverController
"""
self.logger = LoggerClient.LoggerClient(
self.gconfig,
tag = "mistamover_ctl",
name = "top-level mistamover controller",
debug_on = self.debug_on)
self.logger.exportMethods(self)
def startLogServer(self):
"""
Start the logger server process
"""
self.log_server_proc = Daemon.DaemonCtl(self.runLogger,
args = [self.gconfig],
description = "mistamover_log_server")
def stopLogServer(self):
try:
self.log_server_proc.shutdown()
except Exception, ex:
str(ex)