def setUp(self):
super().setUp()
SlaveConfigLoader().configure_defaults(Configuration.singleton())
SlaveConfigLoader().configure_postload(Configuration.singleton())
self.mock_network = self.patch('app.slave.cluster_slave.Network').return_value
self._mock_sys = self.patch('app.slave.cluster_slave.sys')
self.patch('app.util.fs.tar_directories')
def test_configure_postload_sets_protocol_scheme_to_https(self):
mock_config_file = self.patch(
'app.util.conf.base_config_loader.ConfigFile').return_value
mock_config_file.read_config_from_disk.return_value = {
'general': {
'https_cert_file': '/path/to/cert',
'https_key_file': '/path/to/key'
},
'slave': {}
}
config = Configuration.singleton()
config_loader = SlaveConfigLoader()
config_loader.configure_defaults(config)
config_loader.load_from_config_file(config,
config_filename='fake_filename')
config_loader.configure_postload(config)
key = 'protocol_scheme'
expected_stored_protocol_scheme_value = 'https'
actual_stored_protocol_scheme_value = Configuration[key]
self.assertEqual(
expected_stored_protocol_scheme_value,
actual_stored_protocol_scheme_value,
'The configuration value for the key "{}" was expected to be {}:{}, but was {}:{}.'
.format(key, type(expected_stored_protocol_scheme_value),
expected_stored_protocol_scheme_value,
type(actual_stored_protocol_scheme_value),
actual_stored_protocol_scheme_value))
def test_configure_default_sets_protocol_scheme_to_http(self):
mock_config_file = self.patch('app.util.conf.base_config_loader.ConfigFile').return_value
config = Configuration.singleton()
config_loader = SlaveConfigLoader()
config_loader.configure_defaults(config)
key = 'protocol_scheme'
expected_stored_protocol_scheme_value = 'http'
actual_stored_protocol_scheme_value = Configuration[key]
self.assertEqual(expected_stored_protocol_scheme_value, actual_stored_protocol_scheme_value,
'The configuration value for the key "{}" was expected to be {}:{}, but was {}:{}.'.format(
key, type(expected_stored_protocol_scheme_value), expected_stored_protocol_scheme_value,
type(actual_stored_protocol_scheme_value), actual_stored_protocol_scheme_value))
def _initialize_configuration(app_subcommand, config_filename):
"""
Load the default conf values (including subcommand-specific values), then find the conf file and read overrides.
:param app_subcommand: The application subcommand (e.g., master, slave, build)
:type app_subcommand: str
:type config_filename: str
"""
app_subcommand_conf_loaders = {
'master': MasterConfigLoader(),
'slave': SlaveConfigLoader(),
'build': MasterConfigLoader(),
'deploy': DeployConfigLoader(),
'stop': StopConfigLoader(),
}
conf_loader = app_subcommand_conf_loaders.get(
app_subcommand) or BaseConfigLoader()
config = Configuration.singleton()
# First, set the defaults, then load any config from disk, then set additional config values based on the
# base_directory
conf_loader.configure_defaults(config)
config_filename = config_filename or Configuration['config_file']
conf_loader.load_from_config_file(config, config_filename)
conf_loader.configure_postload(config)
_set_secret(config_filename)
def run(self, log_level, master, master_port, slaves, slave_port, num_executors):
"""
'Deploy' can be a vague word, so we should be specific about what this command accomplishes.
This command will:
- Replace the existing binary files in the slaves and master hosts with the binary files running this
command currently. If there is nothing to replace, this command will just place the binary files there.
- Stop all clusterrunner services running on all slaves and the master.
- Start the master and slave services on the master and slave hosts.
- Poll until timeout to validate that the master service has started, and that the slaves have successfully
connected with the master.
:param log_level: the log level at which to do application logging (or None for default log level)
:type log_level: str | None
:param master: the master hostname (no port) to deploy the master service to
:type master: str | None
:param master_port: the port number the master service will listen on
:type master_port: int | None
:param slaves: list of slave hostnames (no ports) to deploy the slave service to
:type slaves: [str] | None
:param slave_port: the port number the slave services will listen on
:type slave_port: int | None
:param num_executors: the number of executors that will be run per slave
:type num_executors: int | None
"""
log.configure_logging(
log_level=log_level or Configuration['log_level'],
log_file=Configuration['log_file'],
simplified_console_logs=True,
)
conf_path = Configuration['config_file']
current_executable = sys.executable
username = getpass.getuser()
slave_config = self._get_loaded_config(conf_path, SlaveConfigLoader())
master_config = self._get_loaded_config(conf_path, MasterConfigLoader())
master = master or slave_config.get('master_hostname')
master_port = master_port or master_config.get('port')
slaves = slaves or master_config.get('slaves')
slave_port = slave_port or slave_config.get('port')
num_executors = num_executors or slave_config.get('num_executors')
clusterrunner_executable_dir = join(os.path.expanduser('~'), '.clusterrunner', 'dist')
clusterrunner_executable = join(clusterrunner_executable_dir, 'clusterrunner')
self._logger.info('Compressing binaries...')
binaries_tar_path = self._binaries_tar(current_executable, Configuration['root_directory'])
self._logger.info('Deploying binaries and confs on master and slaves...')
arguments = [[host, username, current_executable, binaries_tar_path, conf_path] for host in slaves + [master]]
Pool().starmap(self._deploy_binaries_and_conf, arguments)
self._logger.info('Stopping and starting all clusterrunner services...')
self._start_services(master, master_port, slaves, slave_port, num_executors, username, clusterrunner_executable)
self._logger.info('Validating successful deployment...')
master_service_url = '{}:{}'.format(master, master_port)
self._validate_successful_deployment(master_service_url, slaves)
self._logger.info('Deploy SUCCESS to slaves: {}'.format(','.join(slaves)))
def test_configure_default_sets_protocol_scheme_to_http(self):
mock_config_file = self.patch(
'app.util.conf.base_config_loader.ConfigFile').return_value
config = Configuration.singleton()
config_loader = SlaveConfigLoader()
config_loader.configure_defaults(config)
key = 'protocol_scheme'
expected_stored_protocol_scheme_value = 'http'
actual_stored_protocol_scheme_value = Configuration[key]
self.assertEqual(
expected_stored_protocol_scheme_value,
actual_stored_protocol_scheme_value,
'The configuration value for the key "{}" was expected to be {}:{}, but was {}:{}.'
.format(key, type(expected_stored_protocol_scheme_value),
expected_stored_protocol_scheme_value,
type(actual_stored_protocol_scheme_value),
actual_stored_protocol_scheme_value))
def test_configure_postload_sets_protocol_scheme_to_https(self):
mock_config_file = self.patch('app.util.conf.base_config_loader.ConfigFile').return_value
mock_config_file.read_config_from_disk.return_value = {'general': {'https_cert_file': '/path/to/cert',
'https_key_file': '/path/to/key'},
'slave': {}
}
config = Configuration.singleton()
config_loader = SlaveConfigLoader()
config_loader.configure_defaults(config)
config_loader.load_from_config_file(config, config_filename='fake_filename')
config_loader.configure_postload(config)
key = 'protocol_scheme'
expected_stored_protocol_scheme_value = 'https'
actual_stored_protocol_scheme_value = Configuration[key]
self.assertEqual(expected_stored_protocol_scheme_value, actual_stored_protocol_scheme_value,
'The configuration value for the key "{}" was expected to be {}:{}, but was {}:{}.'.format(
key, type(expected_stored_protocol_scheme_value), expected_stored_protocol_scheme_value,
type(actual_stored_protocol_scheme_value), actual_stored_protocol_scheme_value))
def setUp(self):
super().setUp()
SlaveConfigLoader().configure_defaults(Configuration.singleton())
SlaveConfigLoader().configure_postload(Configuration.singleton())