def executeFromFile(self, filename, stdout=None, column_names=True):
""" Some queries cannot run correctly through MySQLdb, so we must use MySQL client instead """
self.logger.debug("SQLCmd.executeFromFile: %s" % filename)
commandLine = self._mysql_cmd[:]
if not column_names: commandLine.append('--skip-column-names')
commandLine += ['-e', "SOURCE " + filename]
commons.run_command(commandLine, stdout=stdout)
def _callLoader(self, table):
"""
Call Qserv loader
"""
self.logger.info("Partition data, create and load table %s", table)
loaderCmd = self.loaderCmdCommonOpts(table)
loaderCmd += ['--css-remove']
if self.dataConfig.duplicatedTables:
loaderCmd += ['--skip-partition']
loaderCmd += ['--chunks-dir={0}'.format(os.path.join(self.tmpDir,
self._out_dirname,
"chunks/",table))]
# include table-specific config if it exists
tableCfg = os.path.join(self.dataConfig.dataDir, table + ".cfg")
if os.path.exists(tableCfg):
loaderCmd += ['--config={0}'.format(tableCfg)]
# WARN emptyChunks.txt might also be intersection of
# all empltyChunkk file: seel with D. Wang and A. Salnikov
if table in self.dataConfig.directors:
loaderCmd += ['--empty-chunks={0}'.format(self._emptyChunksFile)]
loaderCmd += self.loaderCmdCommonArgs(table)
# Use same logging configuration for loader and integration test
# command line, this allow to redirect loader to sys.stdout, sys.stderr
commons.run_command(loaderCmd,
stdout=sys.stdout,
stderr=sys.stderr)
self.logger.info("Partitioned data loaded for table %s", table)
def executeFromFile(self, filename, stdout=None, column_names=True):
""" Some queries cannot run correctly through MySQLdb, so we must use MySQL client instead """
self.logger.debug("SQLCmd.executeFromFile: %s" % filename)
commandLine = self._mysql_cmd[:]
if not column_names: commandLine.append('--skip-column-names')
commandLine += ['-e', "SOURCE " + filename]
commons.run_command(commandLine, stdout_file=stdout)
def _rsync(url, dest_file, username=None):
full_url = "{0}@{1}".format(username, url) if username else url
cmd = ["rsync",
"-avzhe",
"ssh",
full_url,
dest_file]
commons.run_command(cmd, loglevel=logging.WARN)
def gunzip(self, table_name, zipped_data_filename):
# check if the corresponding data file exists
if not os.path.exists(zipped_data_filename):
raise Exception, "File: '%s' not found" % zipped_data_filename
tmp_suffix = ("%s%s" % (table_name,self.dataReader.dataConfig['data-extension']))
tmp_data_file = os.path.join(self._out_dirname,tmp_suffix)
self.logger.info("Unzipping: %s into %s" % (zipped_data_filename, tmp_data_file))
commons.run_command(["gunzip", "-c", zipped_data_filename], stdout_file=tmp_data_file)
return tmp_data_file
def duplicateAndPartitionData(self, table, data_filename):
self.logger.info("Duplicating and partitioning table '%s' from file '%s'\n" % (table, data_filename))
partition_scriptname = "partition.py"
partition_dirname = os.path.join(self._out_dirname,table+"_partition")
if os.path.exists(partition_dirname):
shutil.rmtree(partition_dirname)
os.makedirs(partition_dirname)
schema_filename = os.path.join(self.dataConfig['dataDirName'], table + self.dataConfig['schema-extension'])
data_filename = os.path.join(self.dataConfig['dataDirName'], table + self.dataConfig['data-extension'])
data_filename_cleanup = False
data_filename_zipped = data_filename + self.dataConfig['zip-extension']
if (not os.path.exists(data_filename)) and os.path.exists(data_filename_zipped):
commons.run_command(["gunzip", data_filename_zipped])
data_filename_cleanup = True
if not os.path.exists(data_filename):
raise Exception, "File: %s not found" % data_filename
chunker_scriptname = "makeChunk.py"
chunker_cmd = [
self.config['bin']['python'],
chunker_scriptname,
'--output-dir', partition_dirname,
'--delimiter', self.dataConfig['delimiter'],
'-S', str(self.dataConfig['num-stripes']),
'-s', str(self.dataConfig['num-substripes']),
'--dupe',
'--node-count', str(self.dataConfig['nbNodes']),
'--node=' + str(self.dataConfig['currentNodeID']),
'--chunk-prefix=' + table,
'--theta-name=' + self.dataConfig[table]['ra-fieldname'],
'--phi-name=' + self.dataConfig[table]['decl-fieldname'],
'--schema', schema_filename,
data_filename
]
out = commons.run_command(chunker_cmd)
self.logger.info("Working in DB : %s. LSST %s data duplicated and partitioned : \n %s"
% (self._dbName, table,out))
if data_filename_cleanup:
commons.run_command(["gzip", data_filename])
return partition_dirname
def _runIndex(self):
"""
Run sph-htm-index as step 1 of the duplication process
"""
for table in self._tables:
if os.path.isfile(os.path.join(self._cfgDirname, table + '.cfg')) is False:
self.logger.error("Path to indexing config file not found")
self.logger.info("Running indexer with output for %r to %r" % (table, self._outDirname))
commons.run_command(["sph-htm-index",
"--config-file=" +
os.path.join(self._cfgDirname, table + ".cfg"),
"--config-file=" + os.path.join(self._cfgDirname, "common.cfg"),
"--in=" + os.path.join(self._cfgDirname, table + ".txt"),
"--out.dir=" + os.path.join(self._outDirname, "index/", table)])
def partitionData(self,table, data_filename):
# partition data
partition_scriptname = "partition.py"
partition_dirname = os.path.join(self._out_dirname,table+"_partition")
if os.path.exists(partition_dirname):
shutil.rmtree(partition_dirname)
os.makedirs(partition_dirname)
# python %s -PObject -t 2 -p 4 %s --delimiter '\t' -S 10 -s 2 --output-dir %s" % (self.partition_scriptname, data_filename, partition_dirname
partition_data_cmd = [
partition_scriptname,
'--output-dir', partition_dirname,
'--chunk-prefix', table,
'--theta-column', str(self.dataConfig[table]['ra-column']),
'--phi-column', str(self.dataConfig[table]['decl-column']),
'--num-stripes=%s' % self.dataConfig['num-stripes'],
'--num-sub-stripes=%s' % self.dataConfig['num-substripes'],
'--delimiter', self.dataConfig['delimiter']
]
if self.dataConfig[table]['chunk-column-id'] != None :
partition_data_cmd.extend(['--chunk-column', str(self.dataConfig[table]['chunk-column-id'])])
partition_data_cmd.append(data_filename)
out = commons.run_command(partition_data_cmd)
self.logger.debug("Data for {0}.{1} partitioned (output :{2})".format(self._dbName, table,out))
return partition_dirname
def loadPartitionedData(self,partition_dirname,table):
load_scriptname = "loader.py"
# TODO : remove hard-coded param : qservTest_caseXX_mysql => check why model table already exists in self._dbName
load_partitioned_data_cmd = [
load_scriptname,
'--user=%s' % self.config['mysqld']['user'],
'--password=%s' % self.config['mysqld']['pass'],
'--database=%s' % self._dbName
]
# if (table in self.dataConfig['partitioned-tables']):
# load_partitioned_data_cmd.extend(['--drop-primary-key', 'Overlap'])
load_partitioned_data_cmd.extend( [
"%s:%s" %
("127.0.0.1",self.config['mysqld']['port']),
partition_dirname,
"%s.%s" % (self._dbName, table)
])
# python master/examples/loader.py --verbose -u root -p changeme --database qservTest_case01_qserv -D clrlsst-dbmaster.in2p3.fr:13306 /opt/qserv-dev/tmp/Object_partition/ qservTest_case01_mysql.Object
out = commons.run_command(load_partitioned_data_cmd)
self.logger.info("Partitioned {0} data loaded (stdout : {1})".format(table,out))
def _runIndex(self):
"""
Run sph-htm-index as step 1 of the duplication process
"""
for table in self._tables:
if os.path.isfile(os.path.join(self._cfgDirname, table + '.cfg')) is False:
self.logger.error("Path to indexing config file not found")
self.logger.info("Running indexer with output for %r to %r" % (table, self._outDirname))
commons.run_command(["sph-htm-index",
"--config-file=" +
os.path.join(self._cfgDirname, table + ".cfg"),
"--config-file=" + os.path.join(self._cfgDirname, "common.cfg"),
"--in=" + os.path.join(self._cfgDirname, table + ".txt"),
"--out.dir=" + os.path.join(self._outDirname, "index/", table)])
def dropCssDatabase(self):
script = "qserv-admin.py"
cmd = [script,
"-c",
"localhost:%s" % self.config['zookeeper']['port'],
"-v",
str(self.logger.getEffectiveLevel()),
"-f",
os.path.join(self.config['qserv']['log_dir'],
"qadm-%s.log" % self._dbName)]
with tempfile.NamedTemporaryFile('w+t') as f:
f.write('DROP DATABASE {0};'.format(self._dbName))
f.flush()
commons.run_command(cmd, f.name)
self.logger.info("Drop CSS database: %s",
self._dbName)
def _runDuplicate(self):
"""
Run sph-duplicate to set up partitioned data that the data loader needs
"""
for table in self._tables:
if os.path.isfile(os.path.join(self._cfgDirname, 'common.cfg')) is False:
self.logger.error("Path to duplicator config file not found")
self.logger.info("Running duplicator for table %r" % table)
index_param = os.path.join(self._outDirname, "index", table, "htm_index.bin")
part_index_param = os.path.join(self._outDirname, "index", self._directorTable, "htm_index.bin")
commons.run_command(["sph-duplicate",
"--config-file=" + os.path.join(self._cfgDirname, table + ".cfg"),
"--config-file=" + os.path.join(self._cfgDirname, "common.cfg"),
"--index=" + index_param,
"--part.index=" + part_index_param,
"--out.dir=" + os.path.join(self._outDirname, "chunks/", table)])
def _runDuplicate(self):
"""
Run sph-duplicate to set up partitioned data that the data loader needs
"""
for table in self._tables:
if os.path.isfile(os.path.join(self._cfgDirname, 'common.cfg')) is False:
self.logger.error("Path to duplicator config file not found")
self.logger.info("Running duplicator for table %r" % table)
index_param = os.path.join(self._outDirname, "index", table, "htm_index.bin")
part_index_param = os.path.join(self._outDirname, "index", self._directorTable, "htm_index.bin")
commons.run_command(["sph-duplicate",
"--config-file=" + os.path.join(self._cfgDirname, table + ".cfg"),
"--config-file=" + os.path.join(self._cfgDirname, "common.cfg"),
"--index=" + index_param,
"--part.index=" + part_index_param,
"--out.dir=" + os.path.join(self._outDirname, "chunks/", table)])
def _callLoader(self, table):
"""
Call Qserv loader to load plain-MySQL table
"""
self.logger.info("Create, load table %s", table)
loaderCmd = self.loaderCmdCommonOpts(table)
loaderCmd += ['--no-css',
'--skip-partition',
'--one-table']
loaderCmd += self.loaderCmdCommonArgs(table)
commons.run_command(loaderCmd,
stdout=sys.stdout,
stderr=sys.stderr)
self.logger.info("Partitioned data loaded for table %s", table)
def _callLoader(self, table):
"""
Call Qserv loader to load plain-MySQL table
"""
self.logger.info("Create, load table %s", table)
loaderCmd = self.loaderCmdCommonOpts(table)
loaderCmd += ['--no-css', '--skip-partition', '--one-table']
# include table-specific config if it exists
tableCfg = os.path.join(self.dataConfig.dataDir, table + ".cfg")
if os.path.exists(tableCfg):
loaderCmd += ['--config={0}'.format(tableCfg)]
loaderCmd += self.loaderCmdCommonArgs(table)
commons.run_command(loaderCmd, stdout=sys.stdout, stderr=sys.stderr)
self.logger.info("Partitioned data loaded for table %s", table)
def _callLoader(self, table):
"""
Call Qserv loader
"""
self.logger.info("Partition data, create and load table %s", table)
loaderCmd = self.loaderCmdCommonOpts(table)
loaderCmd += ['--css-remove']
if self.multi_node:
for node in self.nWmgrs:
loaderCmd += ['--worker', node]
if self.dataConfig.duplicatedTables:
loaderCmd += ['--skip-partition']
loaderCmd += [
'--chunks-dir={0}'.format(
os.path.join(self.tmpDir, self._out_dirname, "chunks/",
table))
]
# include table-specific config if it exists
tableCfg = os.path.join(self.dataConfig.dataDir, table + ".cfg")
if os.path.exists(tableCfg):
loaderCmd += ['--config={0}'.format(tableCfg)]
# WARN emptyChunks.txt might also be intersection of
# all empltyChunkk file: seel with D. Wang and A. Salnikov
if table in self.dataConfig.directors:
loaderCmd += ['--empty-chunks={0}'.format(self._emptyChunksFile)]
loaderCmd += self.loaderCmdCommonArgs(table)
# Use same logging configuration for loader and integration test
# command line, this allow to redirect loader to sys.stdout, sys.stderr
commons.run_command(loaderCmd, stdout=sys.stdout, stderr=sys.stderr)
self.logger.info("Partitioned data loaded for table %s", table)
def createCssDatabase(self):
script = "qserv-admin.py"
css_load_cmd = [
script,
"-c",
"localhost:%s" % self.config['zookeeper']['port'],
"-v",
str(self.logger.getEffectiveLevel()),
"-f",
os.path.join(self.config['qserv']['log_dir'], "qadm-%s.log" % self.dataConfig['data-name'])
]
os.chdir(self._in_dirname)
out = commons.run_command(css_load_cmd, "loadMetadata.kv")
self.logger.info("CSS meta successfully loaded for db : %s" % self._dbName)
def _callLoader(self, table):
"""
Call Qserv loader to load plain-MySQL table
"""
self.logger.info("Create, load table %s", table)
loaderCmd = self.loaderCmdCommonOpts(table)
loaderCmd += ['--no-css',
'--skip-partition',
'--one-table']
# include table-specific config if it exists
tableCfg = os.path.join(self.dataConfig.dataDir, table + ".cfg")
if os.path.exists(tableCfg):
loaderCmd += ['--config={0}'.format(tableCfg)]
loaderCmd += self.loaderCmdCommonArgs(table)
commons.run_command(loaderCmd,
stdout=sys.stdout,
stderr=sys.stderr)
self.logger.info("Partitioned data loaded for table %s", table)
def run(self):
"""
Do actual configuration based on parameters provided on command-line-interface and qserv-meta.conf file
This will throw exception if anything goes wrong.
"""
_LOG.info("Qserv configuration tool\n" +
"======================================================================="
)
if commons.status(self.args.qserv_run_dir) not in [commons.NO_STATUS_SCRIPT, commons.DOWN]:
_LOG.fatal(
"Qserv services are still running "
"for this Qserv run directory (%s),"
" stop it before running this script.", self.args.qserv_run_dir)
sys.exit(1)
if configure.INIT in self.cfg_steps:
if os.path.exists(self.args.qserv_run_dir) and os.listdir(self.args.qserv_run_dir):
if self.args.force or configure.user_yes_no_query(
"WARNING : Do you want to erase all configuration data "
"in {0}?".format(self.args.qserv_run_dir)
):
shutil.rmtree(self.args.qserv_run_dir)
else:
_LOG.fatal(
"Terminating Qserv configuration, specify a different configuration directory")
sys.exit(1)
in_meta_config_file = os.path.join(self._in_config_dir, "qserv-meta.conf")
_LOG.info("Creating meta-configuration file: %s", self._meta_config_file)
params_dict = {
'QSERV_RUN_DIR': self.args.qserv_run_dir,
'QSERV_DATA_DIR': self._qserv_data_dir
}
_LOG.info("Store data in: %s" % self._qserv_data_dir)
self._templater.applyOnce(in_meta_config_file, self._meta_config_file, params_dict)
#
#
# Running configuration procedure
#
#
if configure.has_configuration_step(self.cfg_steps):
try:
_LOG.info("Reading meta-configuration file {0}".format(self._meta_config_file))
config = commons.read_config(self._meta_config_file)
# used in templates targets comments
config['qserv']['meta_config_file'] = self._meta_config_file
except configparser.NoOptionError as exc:
_LOG.fatal("Missing option in meta-configuration file: %s", exc)
sys.exit(1)
if configure.DIRTREE in self.cfg_steps:
_LOG.info("Define main directory structure")
configure.update_root_dirs()
configure.update_root_symlinks()
#
# Create Qserv services configuration
# using templates and meta_config_file
#
qserv_run_dir = config['qserv']['qserv_run_dir']
qserv_data_dir = config['qserv']['qserv_data_dir']
if configure.ETC in self.cfg_steps:
_LOG.info(
"Create configuration files in {0}".format(os.path.join(qserv_run_dir, "etc")) +
" and scripts in {0}".format(os.path.join(qserv_run_dir, "tmp"))
)
dest_root = os.path.join(qserv_run_dir)
self._templater.applyAll(self._template_root, dest_root)
# Override default templates
if self._custom_template_root:
self._templater.applyAll(self._custom_template_root, dest_root)
#
# Disable database initialisation scripts if specified by user or if data directory is not empty
#
has_data = (os.path.exists(self._qserv_data_dir) and os.listdir(self._qserv_data_dir))
if self.args.disable_db_init or has_data :
_LOG.info("Remove configuration steps impacting data, %s will remain untouched",
self._qserv_data_dir)
self.cfg_steps = configure.filter_list(self.cfg_steps, configure.DB_COMPONENTS)
script_cfg_steps = configure.intersect_list(self.cfg_steps, configure.COMPONENTS)
if script_cfg_steps:
_LOG.info("Run configuration scripts")
configuration_scripts_dir = os.path.join(
qserv_run_dir, 'tmp', 'configure'
)
if config['qserv']['node_type'] in ['master']:
_LOG.info(
"Master instance: not configuring " +
"{0}".format(configure.WORKER)
)
script_cfg_steps = configure.filter_list(script_cfg_steps, [configure.WORKER])
elif config['qserv']['node_type'] in ['worker']:
_LOG.info(
"Worker instance: not configuring " +
"{0}".format(configure.CZAR)
)
script_cfg_steps = configure.filter_list(script_cfg_steps, [configure.CZAR])
#
#
# Launching configuration scripts
#
#
for comp in script_cfg_steps:
cfg_script = os.path.join(
configuration_scripts_dir, comp + ".sh")
if os.path.isfile(cfg_script):
commons.run_command([cfg_script])
if configure.CSS_WATCHER in self.cfg_steps:
self._template_to_client_config(configure.MYSQL)
if configure.CLIENT in self.cfg_steps:
self._template_to_client_config(configure.QSERV)
def run(self):
"""
Do actual configuration based on parameters provided on command-line-interface and qserv-meta.conf file
This will throw exception if anything goes wrong.
"""
_LOG.info("Qserv configuration tool\n" +
"======================================================================="
)
if commons.status(self.args.qserv_run_dir) not in [commons.NO_STATUS_SCRIPT, commons.DOWN]:
_LOG.fatal(
"Qserv services are still running "
"for this Qserv run directory (%s),"
" stop it before running this script.", self.args.qserv_run_dir)
sys.exit(1)
if configure.INIT in self.args.step_list:
if os.path.exists(self.args.qserv_run_dir):
if self.args.force or configure.user_yes_no_query(
"WARNING : Do you want to erase all configuration data in {0} ?".format(self.args.qserv_run_dir)
):
shutil.rmtree(self.args.qserv_run_dir)
else:
_LOG.fatal(
"Terminating Qserv configuration, specify a different configuration directory")
sys.exit(1)
in_meta_config_file = os.path.join(self._in_config_dir, "qserv-meta.conf")
_LOG.info("Creating meta-configuration file: %s", self._meta_config_file)
params_dict = {
'QSERV_RUN_DIR': self.args.qserv_run_dir,
'QSERV_DATA_DIR': self._qserv_data_dir
}
_LOG.info("Store data in: %s" % self._qserv_data_dir)
configure.apply_tpl_once(
in_meta_config_file, self._meta_config_file, params_dict)
###################################
#
# Running configuration procedure
#
###################################
if configure.has_configuration_step(self.args.step_list):
try:
_LOG.info("Reading meta-configuration file {0}".format(self._meta_config_file))
config = commons.read_config(self._meta_config_file)
# used in templates targets comments
config['qserv']['meta_config_file'] = self._meta_config_file
except ConfigParser.NoOptionError as exc:
_LOG.fatal("Missing option in meta-configuration file: %s", exc)
sys.exit(1)
if configure.DIRTREE in self.args.step_list:
_LOG.info("Define main directory structure")
configure.update_root_dirs()
configure.update_root_symlinks()
##########################################
#
# Creating Qserv services configuration
# using templates and meta_config_file
#
##########################################
qserv_run_dir = config['qserv']['qserv_run_dir']
qserv_data_dir = config['qserv']['qserv_data_dir']
if configure.ETC in self.args.step_list:
_LOG.info(
"Create configuration files in {0}".format(os.path.join(qserv_run_dir, "etc")) +
" and scripts in {0}".format(os.path.join(qserv_run_dir, "tmp"))
)
# TODO: see DM-2580
# in_template_config_dir = os.path.join(self._in_config_dir, "templates")
# out_template_config_dir = os.path.join(self.args.qserv_run_dir, "templates")
# _LOG.info("Copying template configuration from {0} to {1}".format(in_template_config_dir,
# self.args.qserv_run_dir)
# )
# shutil.copytree(in_template_config_dir, out_template_config_dir)
dest_root = os.path.join(qserv_run_dir)
configure.apply_tpl_all( self._template_root, dest_root)
component_cfg_steps = configure.intersect(
self.args.step_list, configure.COMPONENTS)
if len(component_cfg_steps) > 0:
_LOG.info("Run configuration scripts")
configuration_scripts_dir = os.path.join(
qserv_run_dir, 'tmp', 'configure'
)
if config['qserv']['node_type'] in ['master']:
_LOG.info(
"Master instance: not configuring " +
"{0}".format(configure.WORKER)
)
component_cfg_steps.remove(configure.WORKER)
elif config['qserv']['node_type'] in ['worker']:
_LOG.info(
"Worker instance: not configuring " +
"{0}".format(configure.CZAR)
)
component_cfg_steps.remove(configure.CZAR)
component_cfg_steps = configure.keep_data(component_cfg_steps, qserv_data_dir)
for comp in component_cfg_steps:
cfg_script = os.path.join(
configuration_scripts_dir, comp + ".sh")
if os.path.isfile(cfg_script):
commons.run_command([cfg_script])
if configure.CSS_WATCHER in self.args.step_list:
self._template_to_client_config(configure.MYSQL)
if configure.CLIENT in self.args.step_list:
self._template_to_client_config(configure.QSERV)
def _rsync(url, dest_file, username=None):
full_url = "{0}@{1}".format(username, url) if username else url
cmd = ["rsync", "-avzhe", "ssh", full_url, dest_file]
commons.run_command(cmd, loglevel=logging.WARN)
def run(self):
"""
Do actual configuration based on parameters provided on command-line-interface and qserv-meta.conf file
This will throw exception if anything goes wrong.
"""
_LOG.info(
"Qserv configuration tool\n" +
"======================================================================="
)
if commons.status(self.args.qserv_run_dir) not in [
commons.NO_STATUS_SCRIPT, commons.DOWN
]:
_LOG.fatal(
"Qserv services are still running "
"for this Qserv run directory (%s),"
" stop it before running this script.",
self.args.qserv_run_dir)
sys.exit(1)
if configure.INIT in self.cfg_steps:
if os.path.exists(self.args.qserv_run_dir) and os.listdir(
self.args.qserv_run_dir):
if self.args.force or configure.user_yes_no_query(
"WARNING : Do you want to erase all configuration data "
"in {0}?".format(self.args.qserv_run_dir)):
shutil.rmtree(self.args.qserv_run_dir)
else:
_LOG.fatal(
"Terminating Qserv configuration, specify a different configuration directory"
)
sys.exit(1)
in_meta_config_file = os.path.join(self._in_config_dir,
"qserv-meta.conf")
_LOG.info("Creating meta-configuration file: %s",
self._meta_config_file)
params_dict = {
'QSERV_RUN_DIR': self.args.qserv_run_dir,
'QSERV_DATA_DIR': self._qserv_data_dir
}
_LOG.info("Store data in: %s" % self._qserv_data_dir)
self._templater.applyOnce(in_meta_config_file,
self._meta_config_file, params_dict)
#
#
# Running configuration procedure
#
#
if configure.has_configuration_step(self.cfg_steps):
try:
_LOG.info("Reading meta-configuration file {0}".format(
self._meta_config_file))
config = commons.read_config(self._meta_config_file)
# used in templates targets comments
config['qserv']['meta_config_file'] = self._meta_config_file
except configparser.NoOptionError as exc:
_LOG.fatal("Missing option in meta-configuration file: %s",
exc)
sys.exit(1)
if configure.DIRTREE in self.cfg_steps:
_LOG.info("Define main directory structure")
configure.update_root_dirs()
configure.update_root_symlinks()
#
# Create Qserv services configuration
# using templates and meta_config_file
#
qserv_run_dir = config['qserv']['qserv_run_dir']
qserv_data_dir = config['qserv']['qserv_data_dir']
if configure.ETC in self.cfg_steps:
_LOG.info("Create configuration files in {0}".format(
os.path.join(qserv_run_dir, "etc")) +
" and scripts in {0}".format(
os.path.join(qserv_run_dir, "tmp")))
dest_root = os.path.join(qserv_run_dir)
self._templater.applyAll(self._template_root, dest_root)
# Override default templates
if self._custom_template_root:
self._templater.applyAll(self._custom_template_root,
dest_root)
#
# Disable database initialisation scripts if specified by user or if data directory is not empty
#
has_data = (os.path.exists(self._qserv_data_dir)
and os.listdir(self._qserv_data_dir))
if self.args.disable_db_init or has_data:
_LOG.info(
"Remove configuration steps impacting data, %s will remain untouched",
self._qserv_data_dir)
self.cfg_steps = configure.filter_list(self.cfg_steps,
configure.DB_COMPONENTS)
script_cfg_steps = configure.intersect_list(
self.cfg_steps, configure.COMPONENTS)
if script_cfg_steps:
_LOG.info("Run configuration scripts")
configuration_scripts_dir = os.path.join(
qserv_run_dir, 'tmp', 'configure')
if config['qserv']['node_type'] in ['master']:
_LOG.info("Master instance: not configuring " +
"{0}".format(configure.WORKER))
script_cfg_steps = configure.filter_list(
script_cfg_steps, [configure.WORKER])
elif config['qserv']['node_type'] in ['worker']:
_LOG.info("Worker instance: not configuring " +
"{0}".format(configure.CZAR))
script_cfg_steps = configure.filter_list(
script_cfg_steps, [configure.CZAR])
#
#
# Launching configuration scripts
#
#
for comp in script_cfg_steps:
cfg_script = os.path.join(configuration_scripts_dir,
comp + ".sh")
if os.path.isfile(cfg_script):
commons.run_command([cfg_script])
if configure.CSS_WATCHER in self.cfg_steps:
self._template_to_client_config(configure.MYSQL)
if configure.CLIENT in self.cfg_steps:
self._template_to_client_config(configure.QSERV)
def run(self):
"""
Do actual configuration based on parameters provided on command-line-interface and qserv-meta.conf file
This will throw exception if anything goes wrong.
"""
_LOG.info("Qserv configuration tool\n" +
"======================================================================="
)
if commons.status(self.args.qserv_run_dir) not in [commons.NO_STATUS_SCRIPT, commons.DOWN]:
_LOG.fatal(
"Qserv services are still running "
"for this Qserv run directory (%s),"
" stop it before running this script.", self.args.qserv_run_dir)
sys.exit(1)
if configure.INIT in self.args.step_list:
if os.path.exists(self.args.qserv_run_dir):
if self.args.force or configure.user_yes_no_query(
"WARNING : Do you want to erase all configuration data in {0} ?".format(self.args.qserv_run_dir)
):
shutil.rmtree(self.args.qserv_run_dir)
else:
_LOG.fatal(
"Terminating Qserv configuration, specify a different configuration directory")
sys.exit(1)
in_meta_config_file = os.path.join(self._in_config_dir, "qserv-meta.conf")
_LOG.info("Creating meta-configuration file: %s", self._meta_config_file)
params_dict = {
'QSERV_RUN_DIR': self.args.qserv_run_dir,
'QSERV_DATA_DIR': self._qserv_data_dir
}
_LOG.info("Store data in: %s" % self._qserv_data_dir)
configure.apply_tpl_once(
in_meta_config_file, self._meta_config_file, params_dict)
###################################
#
# Running configuration procedure
#
###################################
if configure.has_configuration_step(self.args.step_list):
try:
_LOG.info("Reading meta-configuration file {0}".format(self._meta_config_file))
config = commons.read_config(self._meta_config_file)
# used in templates targets comments
config['qserv']['meta_config_file'] = self._meta_config_file
except ConfigParser.NoOptionError as exc:
_LOG.fatal("Missing option in meta-configuration file: %s", exc)
sys.exit(1)
if configure.DIRTREE in self.args.step_list:
_LOG.info("Define main directory structure")
configure.update_root_dirs()
configure.update_root_symlinks()
##########################################
#
# Creating Qserv services configuration
# using templates and meta_config_file
#
##########################################
qserv_run_dir = config['qserv']['qserv_run_dir']
qserv_data_dir = config['qserv']['qserv_data_dir']
if configure.ETC in self.args.step_list:
_LOG.info(
"Create configuration files in {0}".format(os.path.join(qserv_run_dir, "etc")) +
" and scripts in {0}".format(os.path.join(qserv_run_dir, "tmp"))
)
# TODO: see DM-2580
# in_template_config_dir = os.path.join(self._in_config_dir, "templates")
# out_template_config_dir = os.path.join(self.args.qserv_run_dir, "templates")
# _LOG.info("Copying template configuration from {0} to {1}".format(in_template_config_dir,
# self.args.qserv_run_dir)
# )
# shutil.copytree(in_template_config_dir, out_template_config_dir)
dest_root = os.path.join(qserv_run_dir)
configure.apply_tpl_all( self._template_root, dest_root)
component_cfg_steps = configure.intersect(
self.args.step_list, configure.COMPONENTS)
if len(component_cfg_steps) > 0:
_LOG.info("Run configuration scripts")
configuration_scripts_dir = os.path.join(
qserv_run_dir, 'tmp', 'configure'
)
if config['qserv']['node_type'] in ['master']:
_LOG.info(
"Master instance: not configuring " +
"{0}".format(configure.WORKER)
)
component_cfg_steps.remove(configure.WORKER)
elif config['qserv']['node_type'] in ['worker']:
_LOG.info(
"Worker instance: not configuring " +
"{0}".format(configure.CZAR)
)
component_cfg_steps.remove(configure.CZAR)
component_cfg_steps = configure.keep_data(component_cfg_steps, qserv_data_dir)
for comp in component_cfg_steps:
cfg_script = os.path.join(
configuration_scripts_dir, comp + ".sh")
if os.path.isfile(cfg_script):
commons.run_command([cfg_script])
if configure.CSS_WATCHER in self.args.step_list:
self._template_to_client_config(configure.MYSQL)
if configure.CLIENT in self.args.step_list:
self._template_to_client_config(configure.QSERV)
def main():
args = parseArgs()
logging.basicConfig(format='%(levelname)s: %(message)s', level=args.verbose_level)
logging.info("Qserv configuration tool\n"+
"======================================================================="
)
qserv_dir = os.path.abspath(
os.path.join(
os.path.dirname(os.path.realpath(__file__)),
"..")
)
if configure.PREPARE in args.step_list:
template_config_dir = os.path.join(qserv_dir, "admin")
logging.info("Initializing template configuration in {1} using {0}"
.format(template_config_dir, args.qserv_run_dir)
)
if os.path.exists(args.qserv_run_dir):
if args.force or configure.user_yes_no_query(
"WARNING : Do you want to erase all configuration" +
" data in {0} ?".format(args.qserv_run_dir)
):
shutil.rmtree(args.qserv_run_dir)
else:
logging.info("Stopping Qserv configuration, please specify an other configuration directory")
sys.exit(1)
recursive_copy(template_config_dir, args.qserv_run_dir)
for line in fileinput.input(args.meta_config_file, inplace=1):
print line.replace("run_base_dir =", "run_base_dir = " + args.qserv_run_dir),
def intersect(seq1, seq2):
''' returns subset of seq1 which is contained in seq2 keeping original ordering of items '''
seq2 = set(seq2)
return [item for item in seq1 if item in seq2]
def contains_configuration_step(step_list):
return bool(intersect(step_list, configure.STEP_RUN_LIST))
###################################
#
# Running configuration procedure
#
###################################
if contains_configuration_step(args.step_list):
try:
logging.info("Reading meta-configuration file")
config = commons.read_config(args.meta_config_file)
except ConfigParser.NoOptionError, exc:
logging.fatal("An option is missing in your configuration file: %s", exc)
sys.exit(1)
if configure.DIRTREE in args.step_list:
logging.info("Defining main directory structure")
configure.check_root_dirs()
configure.check_root_symlinks()
##########################################
#
# Creating Qserv services configuration
# using templates and meta_config_file
#
##########################################
run_base_dir = config['qserv']['run_base_dir']
if configure.ETC in args.step_list:
logging.info(
"Creating configuration files in {0}".format(os.path.join(run_base_dir, "etc")) +
" and scripts in {0}".format(os.path.join(run_base_dir, "tmp"))
)
template_root = os.path.join(run_base_dir, "templates", "server")
dest_root = os.path.join(run_base_dir)
configure.apply_templates(
template_root,
dest_root
)
components_to_configure = intersect(args.step_list, configure.COMPONENTS)
if len(components_to_configure) > 0:
logging.info("Running configuration scripts")
configuration_scripts_dir = os.path.join(run_base_dir, 'tmp', 'configure')
if not config['qserv']['node_type'] in ['mono', 'worker']:
logging.info("Service isn't a worker or a mono-node instance : not configuring SciSQL")
components_to_configure.remove('scisql')
for comp in components_to_configure:
script = os.path.join(configuration_scripts_dir, comp+".sh")
commons.run_command([script])
if configure.CLIENT in args.step_list:
template_file = os.path.join(
run_base_dir, "templates", "server", "etc", "qserv-client.conf"
)
cfg_file = os.path.join(
run_base_dir, "etc", "qserv-client.conf"
)
configure.apply_tpl(
template_file,
cfg_file
)
logging.info(
"Client configuration file created : {0}".format(cfg_file)
)
homedir = os.path.expanduser("~")
cfg_link = os.path.join(homedir, ".lsst", "qserv.conf")
if os.lstat(cfg_link):
try:
is_symlink_correct = os.path.samefile(cfg_link, cfg_file)
except os.error:
# link is broken
is_symlink_correct = False
if not is_symlink_correct:
if args.force or configure.user_yes_no_query(
("Do you want to update user configuration file " +
"(currently pointing to {0}) for new run directory?"
.format(os.path.realpath(cfg_link))
)
):
os.remove(cfg_link)
os.symlink(cfg_file, cfg_link)
else:
logging.info("Client configuration unmodified.")
sys.exit(1)
else:
# might need to create directory first
try:
os.makedirs(os.path.join(homedir, ".lsst"))
except os.error:
pass
os.symlink(cfg_file, cfg_link)
logging.info(
"Client configuration is now pointing to : {0}".format(cfg_file)
)
def execute(self, query, output=None, column_names=True, async_timeout=0):
"""Execute query and send result to specified output.
Parameters
----------
query : `str`
Query string.
output : object, optional
Either file object or file name, by default output goes to stdout.
column_names : boolean, optional
If `False` then column names are not printed.
async_timeout : int, optional
If >0 then query will run disconnected, its value gives a timeout
in seconds to wait for query completion.
"""
self.logger.debug("SQLCmd.execute: %s", query)
if async_timeout > 0:
# run SUBMIT command and wait until it finishes
commandLine = self._mysql_cmd[:]
commandLine.append('--skip-column-names')
commandLine += ['-e', "SUBMIT " + query]
self.logger.debug("SQLCmd.execute running SUBMIT query")
try:
data = subprocess.check_output(commandLine)
except subprocess.CalledProcessError as exc:
self.logger.error("SUBMIT failed: %s", exc)
return
# read query ID
try:
qid = int(data.split()[0])
self.logger.debug("SQLCmd.execute query ID = %s", qid)
except Exception:
raise RuntimeError("Failed to read query ID from SUBMIT: %s",
data)
# wait until query completes
query = "SELECT STATE FROM INFORMATION_SCHEMA.PROCESSLIST "\
"WHERE ID = {}".format(qid)
commandLine = self._mysql_cmd[:]
commandLine.append('--skip-column-names')
commandLine += ['-e', query]
self.logger.debug("SQLCmd.execute waiting for query to complete")
end_time = time.time() + async_timeout
while time.time() < end_time:
try:
data = subprocess.check_output(commandLine)
except subprocess.CalledProcessError as exc:
self.logger.error("Async status query failed: %s", exc)
return
status = data.strip()
self.logger.debug("SQLCmd.execute query status = %s", status)
if status == b"COMPLETED":
break
else:
raise RuntimeError("Timeout while waiting for detached query")
# OK, we are here, means query completed, to retrieve its result
# we need different query
query = "SELECT * from qserv_result({})".format(qid)
commandLine = self._mysql_cmd[:]
if not column_names:
commandLine.append('--skip-column-names')
commandLine += ['-e', query]
commons.run_command(commandLine, stdout=output)
