def _create_test_config_file(self, base_dir_sys_path):
"""
Create a temporary conf file just for this test.
:param base_dir_sys_path: Sys path of the base app dir
:type base_dir_sys_path: unicode
:return: The path to the conf file
:rtype: str
"""
# Copy default conf file to tmp location
self._conf_template_path = join(self._clusterrunner_repo_dir, 'conf', 'default_clusterrunner.conf')
# Create the conf file inside base dir so we can clean up the test at the end just by removing the base dir
test_conf_file_path = tempfile.NamedTemporaryFile(dir=base_dir_sys_path).name
shutil.copy(self._conf_template_path, test_conf_file_path)
os.chmod(test_conf_file_path, ConfigFile.CONFIG_FILE_MODE)
conf_file = ConfigFile(test_conf_file_path)
# Set custom conf file values for this test
conf_values_to_set = {
'secret': Secret.get(),
'base_directory': base_dir_sys_path,
'max_log_file_size': 1024 * 5,
}
for conf_key, conf_value in conf_values_to_set.items():
conf_file.write_value(conf_key, conf_value, BASE_CONFIG_FILE_SECTION)
return test_conf_file_path
def _load_section_from_config_file(self, config, config_filename, section):
"""
Load a config file and copy all the values in a particular section to the Configuration singleton
:type config: Configuration
:type config_filename: str
:type section: str
"""
try:
config_parsed = ConfigFile(config_filename).read_config_from_disk()
except FileNotFoundError:
sample_filename = join(config.get('root_directory'), 'conf',
'default_clusterrunner.conf')
fs.create_dir(config.get('base_directory'))
shutil.copy(sample_filename, config_filename)
chmod(config_filename, ConfigFile.CONFIG_FILE_MODE)
config_parsed = ConfigFile(config_filename).read_config_from_disk()
if section not in config_parsed:
raise InvalidConfigError(
'The config file {} does not contain a [{}] section'.format(
config_filename, section))
clusterrunner_config = config_parsed[section]
whitelisted_file_keys = self._get_config_file_whitelisted_keys()
for key in clusterrunner_config:
if key not in whitelisted_file_keys:
raise InvalidConfigError(
'The config file contains an invalid key: {}'.format(key))
value = clusterrunner_config[key]
self._cast_and_set(key, value, config)
def _create_test_config_file(self, base_dir_sys_path: str,
**extra_conf_vals) -> str:
"""
Create a temporary conf file just for this test.
:param base_dir_sys_path: Sys path of the base app dir
:param extra_conf_vals: Optional; additional values to set in the conf file
:return: The path to the conf file
"""
# Copy default conf file to tmp location
self._conf_template_path = join(self._clusterrunner_repo_dir, 'conf',
'default_clusterrunner.conf')
# Create the conf file inside base dir so we can clean up the test at the end just by removing the base dir
test_conf_file_path = tempfile.NamedTemporaryFile(
dir=base_dir_sys_path).name
shutil.copy(self._conf_template_path, test_conf_file_path)
os.chmod(test_conf_file_path, ConfigFile.CONFIG_FILE_MODE)
conf_file = ConfigFile(test_conf_file_path)
# Set custom conf file values for this test
conf_values_to_set = {
'secret': Secret.get(),
'base_directory': base_dir_sys_path,
'max_log_file_size': 1024 * 5,
'database_name': TEST_DB_NAME,
'database_url': TEST_DB_URL
}
conf_values_to_set.update(extra_conf_vals)
for conf_key, conf_value in conf_values_to_set.items():
conf_file.write_value(conf_key, conf_value,
BASE_CONFIG_FILE_SECTION)
return test_conf_file_path
def _set_secret(config_filename):
if 'secret' in Configuration and Configuration['secret'] is not None:
secret = Configuration['secret']
else: # No secret found, generate one and persist it
secret = hashlib.sha512().hexdigest()
conf_file = ConfigFile(config_filename)
conf_file.write_value('secret', secret, BASE_CONFIG_FILE_SECTION)
Secret.set(secret)
def _set_secret(config_filename):
if 'secret' in Configuration and Configuration['secret'] is not None:
secret = Configuration['secret']
else: # No secret found, generate one and persist it
secret = hashlib.sha512().hexdigest()
conf_file = ConfigFile(config_filename)
conf_file.write_value('secret', secret, BASE_CONFIG_FILE_SECTION)
Secret.set(secret)
def _set_secret(config_filename):
if 'secret' in Configuration and Configuration['secret'] is not None:
secret = Configuration['secret']
else: # No secret found, generate one and persist it
secret_length = 128
chars = string.ascii_lowercase + string.digits
secret = ''.join(random.SystemRandom().choice(chars) for _ in range(secret_length))
conf_file = ConfigFile(config_filename)
conf_file.write_value('secret', secret, BASE_CONFIG_FILE_SECTION)
Secret.set(secret)
def _set_secret(config_filename):
if 'secret' in Configuration and Configuration['secret'] is not None:
secret = Configuration['secret']
else: # No secret found, generate one and persist it
secret_length = 128
chars = string.ascii_lowercase + string.digits
secret = ''.join(random.SystemRandom().choice(chars)
for _ in range(secret_length))
conf_file = ConfigFile(config_filename)
conf_file.write_value('secret', secret, BASE_CONFIG_FILE_SECTION)
Secret.set(secret)
def _load_section_from_config_file(self, config, config_filename, section):
"""
Load a config file and copy all the values in a particular section to the Configuration singleton
:type config: Configuration
:type config_filename: str
:type section: str
"""
# Only keys from this list will be loaded from a conf file. If the conf file contains other keys we will
# error to alert the user.
config_key_validation = [
'secret',
'base_directory',
'log_level',
'build_symlink_directory',
'hostname',
'slaves',
'port',
'num_executors',
'master_hostname',
'master_port',
'log_filename',
'max_log_file_size',
'eventlog_filename',
'git_strict_host_key_checking',
'cors_allowed_origins_regex',
]
try:
config_parsed = ConfigFile(config_filename).read_config_from_disk()
except FileNotFoundError:
sample_filename = join(config.get('root_directory'), 'conf',
'default_clusterrunner.conf')
fs.create_dir(config.get('base_directory'))
shutil.copy(sample_filename, config_filename)
chmod(config_filename, ConfigFile.CONFIG_FILE_MODE)
config_parsed = ConfigFile(config_filename).read_config_from_disk()
if section not in config_parsed:
raise _InvalidConfigError(
'The config file {} does not contain a [{}] section'.format(
config_filename, section))
clusterrunner_config = config_parsed[section]
for key in clusterrunner_config:
if key not in config_key_validation:
raise _InvalidConfigError(
'The config file contains an invalid key: {}'.format(key))
value = clusterrunner_config[key]
self._cast_and_set(key, value, config)
def _create_test_config_file(self, conf_values_to_set=None):
"""
Create a temporary conf file just for this test.
:return: The path to the conf file
:rtype: str
"""
# Copy default conf file to tmp location
repo_dir = path.dirname(path.dirname(path.dirname(path.realpath(__file__))))
self._conf_template_path = path.join(repo_dir, 'conf', 'default_clusterrunner.conf')
test_conf_file_path = tempfile.NamedTemporaryFile().name
shutil.copy(self._conf_template_path, test_conf_file_path)
os.chmod(test_conf_file_path, ConfigFile.CONFIG_FILE_MODE)
conf_file = ConfigFile(test_conf_file_path)
# Set custom conf file values for this test
conf_values_to_set = conf_values_to_set or {}
for conf_key, conf_value in conf_values_to_set.items():
conf_file.write_value(conf_key, conf_value, BASE_CONFIG_FILE_SECTION)
return test_conf_file_path
def _create_test_config_file(self, conf_values_to_set=None):
"""
Create a temporary conf file just for this test.
:return: The path to the conf file
:rtype: str
"""
# Copy default conf file to tmp location
repo_dir = path.dirname(
path.dirname(path.dirname(path.dirname(path.realpath(__file__)))))
self._conf_template_path = path.join(repo_dir, 'conf',
'default_clusterrunner.conf')
test_conf_file_path = tempfile.NamedTemporaryFile().name
shutil.copy(self._conf_template_path, test_conf_file_path)
os.chmod(test_conf_file_path, ConfigFile.CONFIG_FILE_MODE)
conf_file = ConfigFile(test_conf_file_path)
# Set custom conf file values for this test
conf_values_to_set = conf_values_to_set or {}
for conf_key, conf_value in conf_values_to_set.items():
conf_file.write_value(conf_key, conf_value,
BASE_CONFIG_FILE_SECTION)
return test_conf_file_path