def update_root_dirs():
config = commons.getConfig()
for (section, option) in (('qserv', 'log_dir'), ('qserv', 'tmp_dir'),
('qserv', 'qserv_data_dir')):
dir = config[section][option]
if not _exists_and_is_writable(dir):
_LOG.fatal("%r is not writable check/update permissions or"
" change config[%r][%r]", dir, section, option)
sys.exit(1)
for suffix in ('etc', 'var', 'var/run',
'var/run/mysqld', 'var/lock/subsys'):
dir = os.path.join(config['qserv']['qserv_run_dir'], suffix)
if not _exists_and_is_writable(dir):
_LOG.fatal("%r is not writable check/update permissions", dir)
sys.exit(1)
# user config
user_config_dir = os.path.join(os.getenv("HOME"), ".lsst")
if not _exists_and_is_writable(user_config_dir):
_LOG.fatal("%r is not writable check/update permissions", dir)
sys.exit(1)
_LOG.info("Qserv directory structure creation succeeded")
def __init__(self, case_id, multi_node, testdata_dir, out_dirname_prefix=None):
self.logger = logging.getLogger(__name__)
self.dataLoader = dict()
self._mode = None
self._dbName = None
self.config = commons.getConfig()
self._case_id = case_id
self._multi_node = multi_node
if not out_dirname_prefix:
out_dirname_prefix = self.config['qserv']['tmp_dir']
self._out_dirname = os.path.join(out_dirname_prefix,
"qservTest_case%s" % case_id)
dataset_dir = Benchmark.getDatasetDir(testdata_dir, case_id)
self._in_dirname = os.path.join(dataset_dir, 'data')
self.dataReader = dataConfig.DataConfig(self._in_dirname)
self._queries_dirname = os.path.join(dataset_dir, "queries")
self.dataDuplicator = dataDuplicator.DataDuplicator(self.dataReader,
self._in_dirname,
self._out_dirname)
def update_root_symlinks():
""" symlinks creation for directories externalised from qserv run dir
i.e. QSERV_RUN_DIR/var/log will be symlinked to config['qserv']['log_dir'] if needed
"""
config = commons.getConfig()
for (section, option, symlink_suffix) in (('qserv', 'log_dir',
os.path.join("var", "log")),
('qserv', 'tmp_dir', 'tmp'),
('qserv', 'qserv_data_dir',
os.path.join("var", "lib"))):
symlink_target = config[section][option]
default_dir = os.path.join(config['qserv']['qserv_run_dir'],
symlink_suffix)
# symlink if target directory is not set to its default value
if symlink_target != default_dir:
if os.path.exists(default_dir):
if os.path.islink(default_dir):
os.unlink(default_dir)
else:
_LOG.fatal(
"Please remove {0} and restart the configuration procedure"
.format(default_dir))
sys.exit(1)
_symlink(symlink_target, default_dir)
_LOG.info("Qserv symlinks creation for externalized directories succeeded")
def update_root_dirs():
config = commons.getConfig()
for (section, option) in (('qserv', 'log_dir'), ('qserv', 'tmp_dir'),
('qserv', 'qserv_data_dir')):
dir = config[section][option]
if not exists_and_is_writable(dir):
_LOG.fatal("%r is not writable check/update permissions or"
" change config[%r][%r]", dir, section, option)
sys.exit(1)
for suffix in ('etc', 'var', 'var/run',
'var/run/mysqld', 'var/lock/subsys'):
dir = os.path.join(config['qserv']['qserv_run_dir'], suffix)
if not exists_and_is_writable(dir):
_LOG.fatal("%r is not writable check/update permissions", dir)
sys.exit(1)
# user config
user_config_dir = os.path.join(os.getenv("HOME"), ".lsst")
if not exists_and_is_writable(user_config_dir):
_LOG.fatal("%r is not writable check/update permissions", dir)
sys.exit(1)
_LOG.info("Qserv directory structure creation succeeded")
def __init__(self, case_id, out_dirname_prefix,
log_file_prefix='qserv-tests',
logging_level=logging.INFO ):
self.logger = logging.getLogger()
self.dataLoader = dict()
self._sqlInterface = dict()
self._mode=None
self._dbName=None
self.config = commons.getConfig()
self.noQservLine = re.compile('[\w\-\."%% ]*-- noQserv')
self._case_id = case_id
self._logFilePrefix = log_file_prefix
if out_dirname_prefix == None :
out_dirname_prefix = self.config['qserv']['tmp_dir']
self._out_dirname = os.path.join(out_dirname_prefix, "qservTest_case%s" % case_id)
# TODO : check existence and "consistency"
if ( not 'testdata_dir' in self.config['qserv'].keys()
or self.config['qserv']['testdata_dir'] == None
or not os.path.isdir(self.config['qserv']['testdata_dir'])
) :
self.logger.critical("Unable to find tests datasets.\n--\n" +
"FOR EUPS USERS :\n"+
"Please run :\n"+
" eups distrib install qserv_testdata\n"+
" setup qserv_testdata\n"+
"FOR NON-EUPS USERS :\n"+
"Please fill 'testdata_dir' value in "+
"~/.lsst/qserv.conf with the path of the directory " +
"containing tests datasets or use --testdata-dir option.\n")
sys.exit(errno.EIO)
else :
self.testdata_dir = self.config['qserv']['testdata_dir']
qserv_tests_dirname = os.path.join(
self.testdata_dir,
"case%s" % self._case_id
)
self._in_dirname = os.path.join(qserv_tests_dirname,'data')
self.dataReader = datareader.DataReader(self._in_dirname, "case%s" % self._case_id)
self._queries_dirname = os.path.join(qserv_tests_dirname,"queries")
def setUpClass(cls):
super(TestIntegration, cls).setUpClass()
TestIntegration.config = commons.getConfig()
TestIntegration.logger = logging.getLogger(__name__)
TestIntegration.modeList = ['mysql', 'qserv']
TestIntegration.loadData = True
if os.environ.get('QSERV_TESTDATA_DIR') is not None:
TestIntegration.testdata_dir = os.path.join(
os.environ.get('QSERV_TESTDATA_DIR'), "datasets")
else:
current_file = os.path.dirname(os.path.realpath(__file__))
fragile_testdata_dir = os.path.join(current_file, os.pardir,
os.pardir, os.pardir,
os.pardir, os.pardir,
"datasets")
TestIntegration.testdata_dir = os.path.abspath(
fragile_testdata_dir)
def setUpClass(cls):
super(TestIntegration, cls).setUpClass()
TestIntegration.config = commons.getConfig()
TestIntegration.logger = logging.getLogger(__name__)
TestIntegration.modeList = ['mysql', 'qserv']
TestIntegration.loadData = True
if os.environ.get('QSERV_TESTDATA_DIR') is not None:
TestIntegration.testdata_dir = os.path.join(os.environ.get('QSERV_TESTDATA_DIR'),
"datasets")
else:
current_file = os.path.dirname(os.path.realpath(__file__))
fragile_testdata_dir = os.path.join(current_file, os.pardir,
os.pardir, os.pardir,
os.pardir, os.pardir,
"datasets"
)
TestIntegration.testdata_dir = os.path.abspath(fragile_testdata_dir)
def update_root_symlinks():
""" symlinks creation for directories externalised from qserv run dir
i.e. QSERV_RUN_DIR/var/log will be symlinked to config['qserv']['log_dir'] if needed
"""
config = commons.getConfig()
for (section, option, symlink_suffix) in (('qserv', 'log_dir', os.path.join("var", "log")),
('qserv', 'tmp_dir', 'tmp'),
('qserv', 'qserv_data_dir', os.path.join("var", "lib"))):
symlink_target = config[section][option]
default_dir = os.path.join(config['qserv']['qserv_run_dir'], symlink_suffix)
# symlink if target directory is not set to its default value
if symlink_target != default_dir:
if os.path.exists(default_dir):
if os.path.islink(default_dir):
os.unlink(default_dir)
else:
log.fatal("Please remove {0} and restart the configuration procedure".format(default_dir))
sys.exit(1)
_symlink(symlink_target, default_dir)
_LOG.info("Qserv symlinks creation for externalized directories succeeded")
def _initTemplateParams(self):
"""
Compute templates parameters from:
- Qserv meta-configuration file
- PATH or other environment variables
"""
if self._templateParams is None:
config = commons.getConfig()
if config['qserv']['node_type'] == 'mono':
comment_mono_node = '#MONO-NODE# '
else:
comment_mono_node = ''
scisql_dir = os.environ.get('SCISQL_DIR')
if scisql_dir is None:
_LOG.fatal("sciSQL install : sciSQL is missing, please install"
"it and set SCISQL_DIR environment variable.")
sys.exit(1)
# find python executable in $PATH
python_bin = "NOT-AVAILABLE"
path = os.environ.get('PATH', '')
for p in path.split(os.pathsep):
python = os.path.join(p, 'python')
if os.access(p, os.X_OK):
python_bin = python
break
self._templateParams = {
'CMSD_MANAGER_PORT': config['xrootd']['cmsd_manager_port'],
'COMMENT_MONO_NODE': comment_mono_node,
'HOME': os.path.expanduser("~"),
'LD_LIBRARY_PATH': os.environ.get('LD_LIBRARY_PATH'),
'LUA_DIR': os.path.join(config['lua']['base_dir']),
'MYSQLD_DATA_DIR':
os.path.join(config['qserv']['qserv_data_dir'],
"mysql"),
# used for mysql-proxy in mono-node
'MYSQLD_HOST': '127.0.0.1',
'MYSQLD_PASSWORD_MONITOR': config['mysqld']['password_monitor'],
'MYSQLD_PASSWORD_ROOT': config['mysqld']['password_root'],
'MYSQLD_PORT': config['mysqld']['port'],
'MYSQLD_SOCK': config['mysqld']['sock'],
'MYSQLD_USER_MONITOR': config['mysqld']['user_monitor'],
'MYSQLD_USER_QSERV': config['mysqld']['user_qserv'],
'MYSQL_DIR': config['mysqld']['base_dir'],
'MYSQLPROXY_DIR': config['mysql_proxy']['base_dir'],
'MYSQLPROXY_PORT': config['mysql_proxy']['port'],
'NODE_TYPE': config['qserv']['node_type'],
'PATH': os.environ.get('PATH'),
'PYTHONPATH': os.environ['PYTHONPATH'],
'PYTHON_BIN': python_bin,
'QSERV_DATA_DIR': config['qserv']['qserv_data_dir'],
'QSERV_DIR': config['qserv']['base_dir'],
'QSERV_LOG_DIR': config['qserv']['log_dir'],
'QSERV_MASTER': config['qserv']['master'],
'QSERV_META_CONFIG_FILE': config['qserv']['meta_config_file'],
'QSERV_PID_DIR': os.path.join(config['qserv']['qserv_run_dir'], "var", "run"),
'QSERV_RUN_DIR': config['qserv']['qserv_run_dir'],
'QSERV_UNIX_USER': getpass.getuser(),
'QSERV_VERSION': self.qserv_version,
'SCISQL_DIR': scisql_dir,
'WMGR_PASSWORD': random_string(string.ascii_letters + string.digits + string.punctuation, 23),
'WMGR_PORT': config['wmgr']['port'],
'WMGR_SECRET_KEY': random_string(string.ascii_letters + string.digits, 57),
'WMGR_USER_NAME': random_string(string.ascii_letters + string.digits, 12),
'XROOTD_ADMIN_DIR': os.path.join(config['qserv']['qserv_run_dir'], 'tmp'),
'XROOTD_DIR': config['xrootd']['base_dir'],
'XROOTD_MANAGER_HOST': config['qserv']['master'],
'XROOTD_PORT': config['xrootd']['xrootd_port'],
}
_LOG.debug("Template input parameters:\n {0}".format(self._templateParams))
def _get_template_params():
""" Compute templates parameters from Qserv meta-configuration file
from PATH or from environment variables for products not needed during build
"""
config = commons.getConfig()
global template_params_dict
if template_params_dict is None:
if config['qserv']['node_type'] == 'mono':
comment_mono_node = '#MONO-NODE# '
else:
comment_mono_node = ''
testdata_dir = os.getenv('QSERV_TESTDATA_DIR', "NOT-AVAILABLE # please set environment variable QSERV_TESTDATA_DIR if needed")
scisql_dir = os.environ.get('SCISQL_DIR')
if scisql_dir is None:
_LOG.fatal("sciSQL install : sciSQL is missing, please install it and set SCISQL_DIR environment variable.")
sys.exit(1)
# find python executable in $PATH
python_bin = "NOT-AVAILABLE"
path = os.environ.get('PATH', '')
for p in path.split(os.pathsep):
python = os.path.join(p, 'python')
if os.access(p, os.X_OK):
python_bin = python
break
params_dict = {
'COMMENT_MONO_NODE' : comment_mono_node,
'PATH': os.environ.get('PATH'),
'LD_LIBRARY_PATH': os.environ.get('LD_LIBRARY_PATH'),
'PYTHON_BIN': python_bin,
'PYTHONPATH': os.environ['PYTHONPATH'],
'QSERV_MASTER': config['qserv']['master'],
'QSERV_DATA_DIR': config['qserv']['qserv_data_dir'],
'QSERV_DIR': config['qserv']['base_dir'],
'QSERV_RUN_DIR': config['qserv']['qserv_run_dir'],
'QSERV_UNIX_USER': getpass.getuser(),
'QSERV_LOG_DIR': config['qserv']['log_dir'],
'QSERV_META_CONFIG_FILE': config['qserv']['meta_config_file'],
'QSERV_PID_DIR': os.path.join(config['qserv']['qserv_run_dir'], "var", "run"),
'QSERV_USER': config['qserv']['user'],
'QSERV_SCRATCH_DIR': config['qserv']['scratch_dir'],
'LUA_DIR': os.path.join(config['lua']['base_dir']),
'MYSQL_DIR': config['mysqld']['base_dir'],
'MYSQLD_DATA_DIR': os.path.join(config['qserv']['qserv_data_dir'], "mysql"),
'MYSQLD_PORT': config['mysqld']['port'],
# used for mysql-proxy in mono-node
'MYSQLD_HOST': '127.0.0.1',
'MYSQLD_SOCK': config['mysqld']['sock'],
'MYSQLD_USER': config['mysqld']['user'],
'MYSQLD_PASS': config['mysqld']['pass'],
'MYSQL_PROXY_PORT': config['mysql_proxy']['port'],
'SCISQL_DIR': scisql_dir,
'XROOTD_DIR': config['xrootd']['base_dir'],
'XROOTD_MANAGER_HOST': config['qserv']['master'],
'XROOTD_PORT': config['xrootd']['xrootd_port'],
'XROOTD_ADMIN_DIR': os.path.join(config['qserv']['qserv_run_dir'], 'tmp'),
'CMSD_MANAGER_PORT': config['xrootd']['cmsd_manager_port'],
'HOME': os.path.expanduser("~"),
'NODE_TYPE': config['qserv']['node_type'],
'WMGR_PORT': config['wmgr']['port'],
'WMGR_USER_NAME': random_string(string.ascii_letters + string.digits, 12),
'WMGR_PASSWORD': random_string(string.ascii_letters + string.digits + string.punctuation, 23),
'WMGR_SECRET_KEY': random_string(string.ascii_letters + string.digits, 57),
}
_LOG.debug("Template input parameters:\n {0}".format(params_dict))
template_params_dict=params_dict
else:
params_dict=template_params_dict
return params_dict
def _initTemplateParams(self):
"""
Compute templates parameters from:
- Qserv meta-configuration file
- PATH or other environment variables
"""
if self._templateParams is None:
config = commons.getConfig()
if config['qserv']['node_type'] == 'mono':
comment_mono_node = '#MONO-NODE# '
else:
comment_mono_node = ''
scisql_dir = os.environ.get('SCISQL_DIR')
if scisql_dir is None:
_LOG.fatal("sciSQL install : sciSQL is missing, please install"
"it and set SCISQL_DIR environment variable.")
sys.exit(1)
# find python executable in $PATH
python_bin = "NOT-AVAILABLE"
path = os.environ.get('PATH', '')
for p in path.split(os.pathsep):
python = os.path.join(p, 'python')
if os.access(p, os.X_OK):
python_bin = python
break
self._templateParams = {
'CMSD_MANAGER_PORT': config['xrootd']['cmsd_manager_port'],
'COMMENT_MONO_NODE': comment_mono_node,
'HOME': os.path.expanduser("~"),
'LD_LIBRARY_PATH': os.environ.get('LD_LIBRARY_PATH'),
'LUA_DIR': os.path.join(config['lua']['base_dir']),
'MYSQLD_DATA_DIR':
os.path.join(config['qserv']['qserv_data_dir'],
"mysql"),
# used for mysql-proxy in mono-node
'MYSQLD_HOST': '127.0.0.1',
'MYSQLD_PASSWORD_MONITOR': config['mysqld']['password_monitor'],
'MYSQLD_PASSWORD_ROOT': config['mysqld']['password_root'],
'MYSQLD_PORT': config['mysqld']['port'],
'MYSQLD_SOCK': config['mysqld']['socket'],
'MYSQLD_USER_MONITOR': config['mysqld']['user_monitor'],
'MYSQLD_USER_QSERV': config['mysqld']['user_qserv'],
'MYSQL_DIR': config['mysqld']['base_dir'],
'MYSQLPROXY_DIR': config['mysql_proxy']['base_dir'],
'MYSQLPROXY_PORT': config['mysql_proxy']['port'],
'NODE_TYPE': config['qserv']['node_type'],
'PATH': os.environ.get('PATH'),
'PYTHONPATH': os.environ['PYTHONPATH'],
'PYTHON_BIN': python_bin,
'QSERV_DATA_DIR': config['qserv']['qserv_data_dir'],
'QSERV_DIR': config['qserv']['base_dir'],
'QSERV_LOG_DIR': config['qserv']['log_dir'],
'QSERV_MASTER': config['qserv']['master'],
'QSERV_META_CONFIG_FILE': config['qserv']['meta_config_file'],
'QSERV_PID_DIR': os.path.join(config['qserv']['qserv_run_dir'], "var", "run"),
'QSERV_RUN_DIR': config['qserv']['qserv_run_dir'],
'QSERV_UNIX_USER': getpass.getuser(),
'QSERV_VERSION': self.qserv_version,
'SCISQL_DIR': scisql_dir,
'WMGR_PASSWORD': random_string(string.ascii_letters + string.digits + string.punctuation, 23),
'WMGR_PORT': config['wmgr']['port'],
'WMGR_SECRET_KEY': random_string(string.ascii_letters + string.digits, 57),
'WMGR_USER_NAME': random_string(string.ascii_letters + string.digits, 12),
'XROOTD_ADMIN_DIR': os.path.join(config['qserv']['qserv_run_dir'], 'tmp'),
'XROOTD_DIR': config['xrootd']['base_dir'],
'XROOTD_MANAGER_HOST': config['qserv']['master'],
'XROOTD_PORT': config['xrootd']['xrootd_port'],
}
_LOG.debug("Template input parameters:\n {0}".format(self._templateParams))
def _get_template_params():
""" Compute templates parameters from Qserv meta-configuration file
from PATH or from environment variables for products not needed during build
"""
config = commons.getConfig()
global template_params_dict
if template_params_dict is None:
if config['qserv']['node_type'] == 'mono':
comment_mono_node = '#MONO-NODE# '
else:
comment_mono_node = ''
testdata_dir = os.getenv(
'QSERV_TESTDATA_DIR',
"NOT-AVAILABLE # please set environment variable QSERV_TESTDATA_DIR if needed"
)
scisql_dir = os.environ.get('SCISQL_DIR')
if scisql_dir is None:
_LOG.fatal(
"sciSQL install : sciSQL is missing, please install it and set SCISQL_DIR environment variable."
)
sys.exit(1)
# find python executable in $PATH
python_bin = "NOT-AVAILABLE"
path = os.environ.get('PATH', '')
for p in path.split(os.pathsep):
python = os.path.join(p, 'python')
if os.access(p, os.X_OK):
python_bin = python
break
params_dict = {
'COMMENT_MONO_NODE':
comment_mono_node,
'PATH':
os.environ.get('PATH'),
'LD_LIBRARY_PATH':
os.environ.get('LD_LIBRARY_PATH'),
'PYTHON_BIN':
python_bin,
'PYTHONPATH':
os.environ['PYTHONPATH'],
'QSERV_MASTER':
config['qserv']['master'],
'QSERV_DATA_DIR':
config['qserv']['qserv_data_dir'],
'QSERV_DIR':
config['qserv']['base_dir'],
'QSERV_RUN_DIR':
config['qserv']['qserv_run_dir'],
'QSERV_UNIX_USER':
getpass.getuser(),
'QSERV_LOG_DIR':
config['qserv']['log_dir'],
'QSERV_META_CONFIG_FILE':
config['qserv']['meta_config_file'],
'QSERV_PID_DIR':
os.path.join(config['qserv']['qserv_run_dir'], "var", "run"),
'QSERV_USER':
config['qserv']['user'],
'LUA_DIR':
os.path.join(config['lua']['base_dir']),
'MYSQL_DIR':
config['mysqld']['base_dir'],
'MYSQLD_DATA_DIR':
os.path.join(config['qserv']['qserv_data_dir'], "mysql"),
'MYSQLD_PORT':
config['mysqld']['port'],
# used for mysql-proxy in mono-node
'MYSQLD_HOST':
'127.0.0.1',
'MYSQLD_SOCK':
config['mysqld']['sock'],
'MYSQLD_USER':
config['mysqld']['user'],
'MYSQLD_PASS':
config['mysqld']['pass'],
'MYSQL_PROXY_PORT':
config['mysql_proxy']['port'],
'SCISQL_DIR':
scisql_dir,
'XROOTD_DIR':
config['xrootd']['base_dir'],
'XROOTD_MANAGER_HOST':
config['qserv']['master'],
'XROOTD_PORT':
config['xrootd']['xrootd_port'],
'XROOTD_ADMIN_DIR':
os.path.join(config['qserv']['qserv_run_dir'], 'tmp'),
'CMSD_MANAGER_PORT':
config['xrootd']['cmsd_manager_port'],
'HOME':
os.path.expanduser("~"),
'NODE_TYPE':
config['qserv']['node_type'],
'WMGR_PORT':
config['wmgr']['port'],
'WMGR_USER_NAME':
random_string(string.ascii_letters + string.digits, 12),
'WMGR_PASSWORD':
random_string(
string.ascii_letters + string.digits + string.punctuation, 23),
'WMGR_SECRET_KEY':
random_string(string.ascii_letters + string.digits, 57),
}
_LOG.debug("Template input parameters:\n {0}".format(params_dict))
template_params_dict = params_dict
else:
params_dict = template_params_dict
return params_dict