def _template_to_symlink(self, filename, symlink):
"""
Use template qserv_prefix/share/qserv/configuration/templates/filename to generate
qserv_run_dir/filename and create a symlink to the latter
Allow to generate symlinks in ~/.lsst to file in a given qserv_run_dir.
@param filename: absolute path to the source template file
@param symlink: absolute path to the created symlink
@return: nothing
"""
template_file = os.path.join(
self._template_root, "etc", filename
)
cfg_file = os.path.join(
self.args.qserv_run_dir, "etc", filename
)
configure.apply_tpl_once(
template_file,
cfg_file
)
_LOG.debug("Client configuration file created: %s", cfg_file)
if os.path.exists(symlink):
try:
is_symlink_correct = os.path.samefile(symlink, cfg_file)
except os.error:
# link is broken
is_symlink_correct = False
if not is_symlink_correct:
if self.args.force or configure.user_yes_no_query(
"Do you want to update symbolic link {0} to {1}?".format(os.path.realpath(symlink),
cfg_file)):
os.remove(symlink)
os.symlink(cfg_file, symlink)
else:
_LOG.fatal("Symbolic link to client " +
"configuration unmodified. Exiting.")
sys.exit(1)
else:
try:
os.remove(symlink)
_LOG.debug("Removing broken symbolic link : %s", symlink)
except os.error:
pass
os.symlink(cfg_file, symlink)
_LOG.info("%s points to: %s", symlink, cfg_file)
def main():
args = parseArgs()
logging.basicConfig(format='%(levelname)s: %(message)s', level=args.verbose_level)
logging.info("Qserv configuration tool\n"+
"======================================================================="
)
qserv_dir = os.path.abspath(
os.path.join(
os.path.dirname(os.path.realpath(__file__)),
"..")
)
if configure.PREPARE in args.step_list:
template_config_dir = os.path.join(qserv_dir, "admin")
logging.info("Initializing template configuration in {1} using {0}"
.format(template_config_dir, args.qserv_run_dir)
)
if os.path.exists(args.qserv_run_dir):
if args.force or configure.user_yes_no_query(
"WARNING : Do you want to erase all configuration" +
" data in {0} ?".format(args.qserv_run_dir)
):
shutil.rmtree(args.qserv_run_dir)
else:
logging.info("Stopping Qserv configuration, please specify an other configuration directory")
sys.exit(1)
recursive_copy(template_config_dir, args.qserv_run_dir)
for line in fileinput.input(args.meta_config_file, inplace=1):
print line.replace("run_base_dir =", "run_base_dir = " + args.qserv_run_dir),
def intersect(seq1, seq2):
''' returns subset of seq1 which is contained in seq2 keeping original ordering of items '''
seq2 = set(seq2)
return [item for item in seq1 if item in seq2]
def contains_configuration_step(step_list):
return bool(intersect(step_list, configure.STEP_RUN_LIST))
###################################
#
# Running configuration procedure
#
###################################
if contains_configuration_step(args.step_list):
try:
logging.info("Reading meta-configuration file")
config = commons.read_config(args.meta_config_file)
except ConfigParser.NoOptionError, exc:
logging.fatal("An option is missing in your configuration file: %s", exc)
sys.exit(1)
if configure.DIRTREE in args.step_list:
logging.info("Defining main directory structure")
configure.check_root_dirs()
configure.check_root_symlinks()
##########################################
#
# Creating Qserv services configuration
# using templates and meta_config_file
#
##########################################
run_base_dir = config['qserv']['run_base_dir']
if configure.ETC in args.step_list:
logging.info(
"Creating configuration files in {0}".format(os.path.join(run_base_dir, "etc")) +
" and scripts in {0}".format(os.path.join(run_base_dir, "tmp"))
)
template_root = os.path.join(run_base_dir, "templates", "server")
dest_root = os.path.join(run_base_dir)
configure.apply_templates(
template_root,
dest_root
)
components_to_configure = intersect(args.step_list, configure.COMPONENTS)
if len(components_to_configure) > 0:
logging.info("Running configuration scripts")
configuration_scripts_dir = os.path.join(run_base_dir, 'tmp', 'configure')
if not config['qserv']['node_type'] in ['mono', 'worker']:
logging.info("Service isn't a worker or a mono-node instance : not configuring SciSQL")
components_to_configure.remove('scisql')
for comp in components_to_configure:
script = os.path.join(configuration_scripts_dir, comp+".sh")
commons.run_command([script])
if configure.CLIENT in args.step_list:
template_file = os.path.join(
run_base_dir, "templates", "server", "etc", "qserv-client.conf"
)
cfg_file = os.path.join(
run_base_dir, "etc", "qserv-client.conf"
)
configure.apply_tpl(
template_file,
cfg_file
)
logging.info(
"Client configuration file created : {0}".format(cfg_file)
)
homedir = os.path.expanduser("~")
cfg_link = os.path.join(homedir, ".lsst", "qserv.conf")
if os.lstat(cfg_link):
try:
is_symlink_correct = os.path.samefile(cfg_link, cfg_file)
except os.error:
# link is broken
is_symlink_correct = False
if not is_symlink_correct:
if args.force or configure.user_yes_no_query(
("Do you want to update user configuration file " +
"(currently pointing to {0}) for new run directory?"
.format(os.path.realpath(cfg_link))
)
):
os.remove(cfg_link)
os.symlink(cfg_file, cfg_link)
else:
logging.info("Client configuration unmodified.")
sys.exit(1)
else:
# might need to create directory first
try:
os.makedirs(os.path.join(homedir, ".lsst"))
except os.error:
pass
os.symlink(cfg_file, cfg_link)
logging.info(
"Client configuration is now pointing to : {0}".format(cfg_file)
)
def run(self):
"""
Do actual configuration based on parameters provided on command-line-interface and qserv-meta.conf file
This will throw exception if anything goes wrong.
"""
_LOG.info("Qserv configuration tool\n" +
"======================================================================="
)
if commons.status(self.args.qserv_run_dir) not in [commons.NO_STATUS_SCRIPT, commons.DOWN]:
_LOG.fatal(
"Qserv services are still running "
"for this Qserv run directory (%s),"
" stop it before running this script.", self.args.qserv_run_dir)
sys.exit(1)
if configure.INIT in self.args.step_list:
if os.path.exists(self.args.qserv_run_dir):
if self.args.force or configure.user_yes_no_query(
"WARNING : Do you want to erase all configuration data in {0} ?".format(self.args.qserv_run_dir)
):
shutil.rmtree(self.args.qserv_run_dir)
else:
_LOG.fatal(
"Terminating Qserv configuration, specify a different configuration directory")
sys.exit(1)
in_meta_config_file = os.path.join(self._in_config_dir, "qserv-meta.conf")
_LOG.info("Creating meta-configuration file: %s", self._meta_config_file)
params_dict = {
'QSERV_RUN_DIR': self.args.qserv_run_dir,
'QSERV_DATA_DIR': self._qserv_data_dir
}
_LOG.info("Store data in: %s" % self._qserv_data_dir)
configure.apply_tpl_once(
in_meta_config_file, self._meta_config_file, params_dict)
###################################
#
# Running configuration procedure
#
###################################
if configure.has_configuration_step(self.args.step_list):
try:
_LOG.info("Reading meta-configuration file {0}".format(self._meta_config_file))
config = commons.read_config(self._meta_config_file)
# used in templates targets comments
config['qserv']['meta_config_file'] = self._meta_config_file
except ConfigParser.NoOptionError as exc:
_LOG.fatal("Missing option in meta-configuration file: %s", exc)
sys.exit(1)
if configure.DIRTREE in self.args.step_list:
_LOG.info("Define main directory structure")
configure.update_root_dirs()
configure.update_root_symlinks()
##########################################
#
# Creating Qserv services configuration
# using templates and meta_config_file
#
##########################################
qserv_run_dir = config['qserv']['qserv_run_dir']
qserv_data_dir = config['qserv']['qserv_data_dir']
if configure.ETC in self.args.step_list:
_LOG.info(
"Create configuration files in {0}".format(os.path.join(qserv_run_dir, "etc")) +
" and scripts in {0}".format(os.path.join(qserv_run_dir, "tmp"))
)
# TODO: see DM-2580
# in_template_config_dir = os.path.join(self._in_config_dir, "templates")
# out_template_config_dir = os.path.join(self.args.qserv_run_dir, "templates")
# _LOG.info("Copying template configuration from {0} to {1}".format(in_template_config_dir,
# self.args.qserv_run_dir)
# )
# shutil.copytree(in_template_config_dir, out_template_config_dir)
dest_root = os.path.join(qserv_run_dir)
configure.apply_tpl_all( self._template_root, dest_root)
component_cfg_steps = configure.intersect(
self.args.step_list, configure.COMPONENTS)
if len(component_cfg_steps) > 0:
_LOG.info("Run configuration scripts")
configuration_scripts_dir = os.path.join(
qserv_run_dir, 'tmp', 'configure'
)
if config['qserv']['node_type'] in ['master']:
_LOG.info(
"Master instance: not configuring " +
"{0}".format(configure.WORKER)
)
component_cfg_steps.remove(configure.WORKER)
elif config['qserv']['node_type'] in ['worker']:
_LOG.info(
"Worker instance: not configuring " +
"{0}".format(configure.CZAR)
)
component_cfg_steps.remove(configure.CZAR)
component_cfg_steps = configure.keep_data(component_cfg_steps, qserv_data_dir)
for comp in component_cfg_steps:
cfg_script = os.path.join(
configuration_scripts_dir, comp + ".sh")
if os.path.isfile(cfg_script):
commons.run_command([cfg_script])
if configure.CSS_WATCHER in self.args.step_list:
self._template_to_client_config(configure.MYSQL)
if configure.CLIENT in self.args.step_list:
self._template_to_client_config(configure.QSERV)
def run(self):
"""
Do actual configuration based on parameters provided on command-line-interface and qserv-meta.conf file
This will throw exception if anything goes wrong.
"""
_LOG.info("Qserv configuration tool\n" +
"======================================================================="
)
if commons.status(self.args.qserv_run_dir) not in [commons.NO_STATUS_SCRIPT, commons.DOWN]:
_LOG.fatal(
"Qserv services are still running "
"for this Qserv run directory (%s),"
" stop it before running this script.", self.args.qserv_run_dir)
sys.exit(1)
if configure.INIT in self.args.step_list:
if os.path.exists(self.args.qserv_run_dir):
if self.args.force or configure.user_yes_no_query(
"WARNING : Do you want to erase all configuration data in {0} ?".format(self.args.qserv_run_dir)
):
shutil.rmtree(self.args.qserv_run_dir)
else:
_LOG.fatal(
"Terminating Qserv configuration, specify a different configuration directory")
sys.exit(1)
in_meta_config_file = os.path.join(self._in_config_dir, "qserv-meta.conf")
_LOG.info("Creating meta-configuration file: %s", self._meta_config_file)
params_dict = {
'QSERV_RUN_DIR': self.args.qserv_run_dir,
'QSERV_DATA_DIR': self._qserv_data_dir
}
_LOG.info("Store data in: %s" % self._qserv_data_dir)
configure.apply_tpl_once(
in_meta_config_file, self._meta_config_file, params_dict)
###################################
#
# Running configuration procedure
#
###################################
if configure.has_configuration_step(self.args.step_list):
try:
_LOG.info("Reading meta-configuration file {0}".format(self._meta_config_file))
config = commons.read_config(self._meta_config_file)
# used in templates targets comments
config['qserv']['meta_config_file'] = self._meta_config_file
except ConfigParser.NoOptionError as exc:
_LOG.fatal("Missing option in meta-configuration file: %s", exc)
sys.exit(1)
if configure.DIRTREE in self.args.step_list:
_LOG.info("Define main directory structure")
configure.update_root_dirs()
configure.update_root_symlinks()
##########################################
#
# Creating Qserv services configuration
# using templates and meta_config_file
#
##########################################
qserv_run_dir = config['qserv']['qserv_run_dir']
qserv_data_dir = config['qserv']['qserv_data_dir']
if configure.ETC in self.args.step_list:
_LOG.info(
"Create configuration files in {0}".format(os.path.join(qserv_run_dir, "etc")) +
" and scripts in {0}".format(os.path.join(qserv_run_dir, "tmp"))
)
# TODO: see DM-2580
# in_template_config_dir = os.path.join(self._in_config_dir, "templates")
# out_template_config_dir = os.path.join(self.args.qserv_run_dir, "templates")
# _LOG.info("Copying template configuration from {0} to {1}".format(in_template_config_dir,
# self.args.qserv_run_dir)
# )
# shutil.copytree(in_template_config_dir, out_template_config_dir)
dest_root = os.path.join(qserv_run_dir)
configure.apply_tpl_all( self._template_root, dest_root)
component_cfg_steps = configure.intersect(
self.args.step_list, configure.COMPONENTS)
if len(component_cfg_steps) > 0:
_LOG.info("Run configuration scripts")
configuration_scripts_dir = os.path.join(
qserv_run_dir, 'tmp', 'configure'
)
if config['qserv']['node_type'] in ['master']:
_LOG.info(
"Master instance: not configuring " +
"{0}".format(configure.WORKER)
)
component_cfg_steps.remove(configure.WORKER)
elif config['qserv']['node_type'] in ['worker']:
_LOG.info(
"Worker instance: not configuring " +
"{0}".format(configure.CZAR)
)
component_cfg_steps.remove(configure.CZAR)
component_cfg_steps = configure.keep_data(component_cfg_steps, qserv_data_dir)
for comp in component_cfg_steps:
cfg_script = os.path.join(
configuration_scripts_dir, comp + ".sh")
if os.path.isfile(cfg_script):
commons.run_command([cfg_script])
if configure.CSS_WATCHER in self.args.step_list:
self._template_to_client_config(configure.MYSQL)
if configure.CLIENT in self.args.step_list:
self._template_to_client_config(configure.QSERV)
def run(self):
"""
Do actual configuration based on parameters provided on command-line-interface and qserv-meta.conf file
This will throw exception if anything goes wrong.
"""
_LOG.info(
"Qserv configuration tool\n" +
"======================================================================="
)
if commons.status(self.args.qserv_run_dir) not in [
commons.NO_STATUS_SCRIPT, commons.DOWN
]:
_LOG.fatal(
"Qserv services are still running "
"for this Qserv run directory (%s),"
" stop it before running this script.",
self.args.qserv_run_dir)
sys.exit(1)
if configure.INIT in self.cfg_steps:
if os.path.exists(self.args.qserv_run_dir) and os.listdir(
self.args.qserv_run_dir):
if self.args.force or configure.user_yes_no_query(
"WARNING : Do you want to erase all configuration data "
"in {0}?".format(self.args.qserv_run_dir)):
shutil.rmtree(self.args.qserv_run_dir)
else:
_LOG.fatal(
"Terminating Qserv configuration, specify a different configuration directory"
)
sys.exit(1)
in_meta_config_file = os.path.join(self._in_config_dir,
"qserv-meta.conf")
_LOG.info("Creating meta-configuration file: %s",
self._meta_config_file)
params_dict = {
'QSERV_RUN_DIR': self.args.qserv_run_dir,
'QSERV_DATA_DIR': self._qserv_data_dir
}
_LOG.info("Store data in: %s" % self._qserv_data_dir)
self._templater.applyOnce(in_meta_config_file,
self._meta_config_file, params_dict)
#
#
# Running configuration procedure
#
#
if configure.has_configuration_step(self.cfg_steps):
try:
_LOG.info("Reading meta-configuration file {0}".format(
self._meta_config_file))
config = commons.read_config(self._meta_config_file)
# used in templates targets comments
config['qserv']['meta_config_file'] = self._meta_config_file
except configparser.NoOptionError as exc:
_LOG.fatal("Missing option in meta-configuration file: %s",
exc)
sys.exit(1)
if configure.DIRTREE in self.cfg_steps:
_LOG.info("Define main directory structure")
configure.update_root_dirs()
configure.update_root_symlinks()
#
# Create Qserv services configuration
# using templates and meta_config_file
#
qserv_run_dir = config['qserv']['qserv_run_dir']
qserv_data_dir = config['qserv']['qserv_data_dir']
if configure.ETC in self.cfg_steps:
_LOG.info("Create configuration files in {0}".format(
os.path.join(qserv_run_dir, "etc")) +
" and scripts in {0}".format(
os.path.join(qserv_run_dir, "tmp")))
dest_root = os.path.join(qserv_run_dir)
self._templater.applyAll(self._template_root, dest_root)
# Override default templates
if self._custom_template_root:
self._templater.applyAll(self._custom_template_root,
dest_root)
#
# Disable database initialisation scripts if specified by user or if data directory is not empty
#
has_data = (os.path.exists(self._qserv_data_dir)
and os.listdir(self._qserv_data_dir))
if self.args.disable_db_init or has_data:
_LOG.info(
"Remove configuration steps impacting data, %s will remain untouched",
self._qserv_data_dir)
self.cfg_steps = configure.filter_list(self.cfg_steps,
configure.DB_COMPONENTS)
script_cfg_steps = configure.intersect_list(
self.cfg_steps, configure.COMPONENTS)
if script_cfg_steps:
_LOG.info("Run configuration scripts")
configuration_scripts_dir = os.path.join(
qserv_run_dir, 'tmp', 'configure')
if config['qserv']['node_type'] in ['master']:
_LOG.info("Master instance: not configuring " +
"{0}".format(configure.WORKER))
script_cfg_steps = configure.filter_list(
script_cfg_steps, [configure.WORKER])
elif config['qserv']['node_type'] in ['worker']:
_LOG.info("Worker instance: not configuring " +
"{0}".format(configure.CZAR))
script_cfg_steps = configure.filter_list(
script_cfg_steps, [configure.CZAR])
#
#
# Launching configuration scripts
#
#
for comp in script_cfg_steps:
cfg_script = os.path.join(configuration_scripts_dir,
comp + ".sh")
if os.path.isfile(cfg_script):
commons.run_command([cfg_script])
if configure.CSS_WATCHER in self.cfg_steps:
self._template_to_client_config(configure.MYSQL)
if configure.CLIENT in self.cfg_steps:
self._template_to_client_config(configure.QSERV)
def run(self):
"""
Do actual configuration based on parameters provided on command-line-interface and qserv-meta.conf file
This will throw exception if anything goes wrong.
"""
_LOG.info("Qserv configuration tool\n" +
"======================================================================="
)
if commons.status(self.args.qserv_run_dir) not in [commons.NO_STATUS_SCRIPT, commons.DOWN]:
_LOG.fatal(
"Qserv services are still running "
"for this Qserv run directory (%s),"
" stop it before running this script.", self.args.qserv_run_dir)
sys.exit(1)
if configure.INIT in self.cfg_steps:
if os.path.exists(self.args.qserv_run_dir) and os.listdir(self.args.qserv_run_dir):
if self.args.force or configure.user_yes_no_query(
"WARNING : Do you want to erase all configuration data "
"in {0}?".format(self.args.qserv_run_dir)
):
shutil.rmtree(self.args.qserv_run_dir)
else:
_LOG.fatal(
"Terminating Qserv configuration, specify a different configuration directory")
sys.exit(1)
in_meta_config_file = os.path.join(self._in_config_dir, "qserv-meta.conf")
_LOG.info("Creating meta-configuration file: %s", self._meta_config_file)
params_dict = {
'QSERV_RUN_DIR': self.args.qserv_run_dir,
'QSERV_DATA_DIR': self._qserv_data_dir
}
_LOG.info("Store data in: %s" % self._qserv_data_dir)
self._templater.applyOnce(in_meta_config_file, self._meta_config_file, params_dict)
#
#
# Running configuration procedure
#
#
if configure.has_configuration_step(self.cfg_steps):
try:
_LOG.info("Reading meta-configuration file {0}".format(self._meta_config_file))
config = commons.read_config(self._meta_config_file)
# used in templates targets comments
config['qserv']['meta_config_file'] = self._meta_config_file
except configparser.NoOptionError as exc:
_LOG.fatal("Missing option in meta-configuration file: %s", exc)
sys.exit(1)
if configure.DIRTREE in self.cfg_steps:
_LOG.info("Define main directory structure")
configure.update_root_dirs()
configure.update_root_symlinks()
#
# Create Qserv services configuration
# using templates and meta_config_file
#
qserv_run_dir = config['qserv']['qserv_run_dir']
qserv_data_dir = config['qserv']['qserv_data_dir']
if configure.ETC in self.cfg_steps:
_LOG.info(
"Create configuration files in {0}".format(os.path.join(qserv_run_dir, "etc")) +
" and scripts in {0}".format(os.path.join(qserv_run_dir, "tmp"))
)
dest_root = os.path.join(qserv_run_dir)
self._templater.applyAll(self._template_root, dest_root)
# Override default templates
if self._custom_template_root:
self._templater.applyAll(self._custom_template_root, dest_root)
#
# Disable database initialisation scripts if specified by user or if data directory is not empty
#
has_data = (os.path.exists(self._qserv_data_dir) and os.listdir(self._qserv_data_dir))
if self.args.disable_db_init or has_data :
_LOG.info("Remove configuration steps impacting data, %s will remain untouched",
self._qserv_data_dir)
self.cfg_steps = configure.filter_list(self.cfg_steps, configure.DB_COMPONENTS)
script_cfg_steps = configure.intersect_list(self.cfg_steps, configure.COMPONENTS)
if script_cfg_steps:
_LOG.info("Run configuration scripts")
configuration_scripts_dir = os.path.join(
qserv_run_dir, 'tmp', 'configure'
)
if config['qserv']['node_type'] in ['master']:
_LOG.info(
"Master instance: not configuring " +
"{0}".format(configure.WORKER)
)
script_cfg_steps = configure.filter_list(script_cfg_steps, [configure.WORKER])
elif config['qserv']['node_type'] in ['worker']:
_LOG.info(
"Worker instance: not configuring " +
"{0}".format(configure.CZAR)
)
script_cfg_steps = configure.filter_list(script_cfg_steps, [configure.CZAR])
#
#
# Launching configuration scripts
#
#
for comp in script_cfg_steps:
cfg_script = os.path.join(
configuration_scripts_dir, comp + ".sh")
if os.path.isfile(cfg_script):
commons.run_command([cfg_script])
if configure.CSS_WATCHER in self.cfg_steps:
self._template_to_client_config(configure.MYSQL)
if configure.CLIENT in self.cfg_steps:
self._template_to_client_config(configure.QSERV)