def setUp(self):
super().setUp()
self._set_up_safe_guards()
# Reset singletons so that they get recreated for every test that uses them.
Configuration.reset_singleton()
UnhandledExceptionHandler.reset_singleton()
# Explicitly initialize UnhandledExceptionHandler singleton here (on the main thread) since it sets up signal
# handlers that must execute on the main thread.
UnhandledExceptionHandler.singleton()
MasterConfigLoader().configure_defaults(Configuration.singleton())
MasterConfigLoader().configure_postload(Configuration.singleton())
self.patch('app.util.conf.master_config_loader.MasterConfigLoader.load_from_config_file')
# Configure logging to go to stdout. This makes debugging easier by allowing us to see logs for failed tests.
log.configure_logging('DEBUG')
# Then stub out configure_logging so we don't end up logging to real files during testing.
self.patch('app.util.log.configure_logging')
# Set up TestHandler. This allows asserting on log messages in tests.
self.log_handler = logbook.TestHandler(bubble=True)
self.log_handler.push_application()
self._base_setup_called = True
def setUp(self):
super().setUp()
SlaveConfigLoader().configure_defaults(Configuration.singleton())
SlaveConfigLoader().configure_postload(Configuration.singleton())
self.mock_network = self.patch('app.slave.cluster_slave.Network').return_value
self._mock_sys = self.patch('app.slave.cluster_slave.sys')
self.patch('app.util.fs.tar_directories')
def setUp(self):
super().setUp()
self.addCleanup(patch.stopall)
self._patched_items = {}
self._blacklist_methods_not_allowed_in_unit_tests()
# Stub out a few library dependencies that launch subprocesses.
self.patch(
'app.util.autoversioning.get_version').return_value = '0.0.0'
self.patch('app.util.conf.base_config_loader.platform.node'
).return_value = self._fake_hostname
if self._do_network_mocks:
# requests.Session() also makes some subprocess calls on instantiation.
self.patch('app.util.network.requests.Session')
# Stub out Network.are_hosts_same() call with a simple string comparison.
self.patch('app.util.network.Network.are_hosts_same',
new=lambda host_a, host_b: host_a == host_b)
# Reset singletons so that they get recreated for every test that uses them.
Configuration.reset_singleton()
UnhandledExceptionHandler.reset_singleton()
SlaveRegistry.reset_singleton()
# Explicitly initialize UnhandledExceptionHandler singleton here (on the main thread) since it sets up signal
# handlers that must execute on the main thread.
UnhandledExceptionHandler.singleton()
MasterConfigLoader().configure_defaults(Configuration.singleton())
MasterConfigLoader().configure_postload(Configuration.singleton())
self.patch(
'app.util.conf.master_config_loader.MasterConfigLoader.load_from_config_file'
)
# Reset counters
Slave._slave_id_counter = Counter()
Build._build_id_counter = Counter()
analytics._event_id_generator = Counter()
# Configure logging to go to stdout. This makes debugging easier by allowing us to see logs for failed tests.
log.configure_logging('DEBUG')
# Then stub out configure_logging so we don't end up logging to real files during testing.
self.patch('app.util.log.configure_logging')
# Set up TestHandler. This allows asserting on log messages in tests.
self.log_handler = logbook.TestHandler(bubble=True)
self.log_handler.push_application()
self._base_setup_called = True
def _initialize_configuration(app_subcommand, config_filename):
"""
Load the default conf values (including subcommand-specific values), then find the conf file and read overrides.
:param app_subcommand: The application subcommand (e.g., master, slave, build)
:type app_subcommand: str
:type config_filename: str
"""
app_subcommand_conf_loaders = {
'master': MasterConfigLoader(),
'slave': SlaveConfigLoader(),
'build': MasterConfigLoader(),
'deploy': DeployConfigLoader(),
'stop': StopConfigLoader(),
}
conf_loader = app_subcommand_conf_loaders.get(
app_subcommand) or BaseConfigLoader()
config = Configuration.singleton()
# First, set the defaults, then load any config from disk, then set additional config values based on the
# base_directory
conf_loader.configure_defaults(config)
config_filename = config_filename or Configuration['config_file']
conf_loader.load_from_config_file(config, config_filename)
conf_loader.configure_postload(config)
_set_secret(config_filename)
def test_configure_postload_sets_protocol_scheme_to_https(self):
mock_config_file = self.patch(
'app.util.conf.base_config_loader.ConfigFile').return_value
mock_config_file.read_config_from_disk.return_value = {
'general': {
'https_cert_file': '/path/to/cert',
'https_key_file': '/path/to/key'
},
'slave': {}
}
config = Configuration.singleton()
config_loader = SlaveConfigLoader()
config_loader.configure_defaults(config)
config_loader.load_from_config_file(config,
config_filename='fake_filename')
config_loader.configure_postload(config)
key = 'protocol_scheme'
expected_stored_protocol_scheme_value = 'https'
actual_stored_protocol_scheme_value = Configuration[key]
self.assertEqual(
expected_stored_protocol_scheme_value,
actual_stored_protocol_scheme_value,
'The configuration value for the key "{}" was expected to be {}:{}, but was {}:{}.'
.format(key, type(expected_stored_protocol_scheme_value),
expected_stored_protocol_scheme_value,
type(actual_stored_protocol_scheme_value),
actual_stored_protocol_scheme_value))
def _initialize_configuration(app_subcommand, config_filename):
"""
Load the default conf values (including subcommand-specific values), then find the conf file and read overrides.
:param app_subcommand: The application subcommand (e.g., master, slave, build)
:type app_subcommand: str
:type config_filename: str
"""
app_subcommand_conf_loaders = {
'master': MasterConfigLoader(),
'slave': SlaveConfigLoader(),
'build': MasterConfigLoader(),
'deploy': DeployConfigLoader(),
'stop': StopConfigLoader(),
}
conf_loader = app_subcommand_conf_loaders.get(app_subcommand) or BaseConfigLoader()
config = Configuration.singleton()
# First, set the defaults, then load any config from disk, then set additional config values based on the
# base_directory
conf_loader.configure_defaults(config)
config_filename = config_filename or Configuration['config_file']
conf_loader.load_from_config_file(config, config_filename)
conf_loader.configure_postload(config)
_set_secret(config_filename)
def _assert_conf_values_match_expected(self, expected_conf_values):
conf = Configuration.singleton()
for conf_key, expected_conf_value in expected_conf_values.items():
self.assertEqual(expected_conf_value, conf.get(conf_key),
'Actual conf value via get() should match expected.')
self.assertEqual(expected_conf_value, Configuration[conf_key],
'Actual conf value via keyed access should match expected.')
def setUp(self):
super().setUp()
self.addCleanup(patch.stopall)
self._patched_items = {}
self._blacklist_methods_not_allowed_in_unit_tests()
# Stub out a few library dependencies that launch subprocesses.
self.patch('app.util.autoversioning.get_version').return_value = '0.0.0'
self.patch('app.util.conf.base_config_loader.platform.node').return_value = self._fake_hostname
if self._do_network_mocks:
# requests.Session() also makes some subprocess calls on instantiation.
self.patch('app.util.network.requests.Session')
# Stub out Network.are_hosts_same() call with a simple string comparison.
self.patch('app.util.network.Network.are_hosts_same', new=lambda host_a, host_b: host_a == host_b)
# Reset singletons so that they get recreated for every test that uses them.
Configuration.reset_singleton()
UnhandledExceptionHandler.reset_singleton()
SlaveRegistry.reset_singleton()
# Explicitly initialize UnhandledExceptionHandler singleton here (on the main thread) since it sets up signal
# handlers that must execute on the main thread.
UnhandledExceptionHandler.singleton()
MasterConfigLoader().configure_defaults(Configuration.singleton())
MasterConfigLoader().configure_postload(Configuration.singleton())
self.patch('app.util.conf.master_config_loader.MasterConfigLoader.load_from_config_file')
# Reset counters
Slave._slave_id_counter = Counter()
Build._build_id_counter = Counter()
analytics._event_id_generator = Counter()
# Configure logging to go to stdout. This makes debugging easier by allowing us to see logs for failed tests.
log.configure_logging('DEBUG')
# Then stub out configure_logging so we don't end up logging to real files during testing.
self.patch('app.util.log.configure_logging')
# Set up TestHandler. This allows asserting on log messages in tests.
self.log_handler = logbook.TestHandler(bubble=True)
self.log_handler.push_application()
self._base_setup_called = True
def _assert_conf_values_match_expected(self, expected_conf_values):
conf = Configuration.singleton()
for conf_key, expected_conf_value in expected_conf_values.items():
self.assertEqual(
expected_conf_value, conf.get(conf_key),
'Actual conf value via get() should match expected.')
self.assertEqual(
expected_conf_value, Configuration[conf_key],
'Actual conf value via keyed access should match expected.')
def test_error_is_raised_when_conf_file_contains_nonexistent_or_nonwhitelisted_key(self, key, parsed_val):
mock_config_file = self.patch('app.util.conf.base_config_loader.ConfigFile').return_value
mock_config_file.read_config_from_disk.return_value = {'general': {key: parsed_val}}
config = Configuration.singleton()
config_loader = _FakeConfigLoader()
config_loader.configure_defaults(config)
with self.assertRaises(InvalidConfigError):
config_loader.load_from_config_file(config, config_filename='fake_filename')
def _get_loaded_config(self, conf_file_path, conf_loader):
"""
:param conf_file_path: path to the configuration file
:type conf_file_path: str
:type conf_loader: BaseConfigLoader
:rtype: Configuration
"""
config = Configuration(as_instance=True)
conf_loader.configure_defaults(config)
conf_loader.load_from_config_file(config, conf_file_path)
conf_loader.configure_postload(config)
return config
def test_list_type_conf_file_values_are_correctly_converted_to_lists(self):
conf = Configuration.singleton()
conf.set('some_list', ['localhost']) # The previous conf value determines the expected type: a list.
conf_file_value = 'my-lonely-slave' # ConfigObj parses value to a string type if only one element is specified.
config_loader = BaseConfigLoader()
config_loader._cast_and_set('some_list', conf_file_value, conf)
expected_conf_setting = [conf_file_value]
self.assertListEqual(conf.get('some_list'), expected_conf_setting,
'The config loader should convert string values into single element lists for conf keys '
'that are expected to be of type list.')
def test_all_datatypes_can_be_overridden_by_value_in_file(self, key, parsed_val, expected_stored_conf_val):
mock_config_file = self.patch('app.util.conf.base_config_loader.ConfigFile').return_value
mock_config_file.read_config_from_disk.return_value = {'general': {key: parsed_val}}
config = Configuration.singleton()
config_loader = _FakeConfigLoader()
config_loader.configure_defaults(config)
config_loader.load_from_config_file(config, config_filename='fake_filename')
actual_stored_conf_val = Configuration[key]
self.assertEqual(expected_stored_conf_val, actual_stored_conf_val,
'The configuration value for the key "{}" was expected to be {}:{}, but was {}:{}.'.format(
key, type(expected_stored_conf_val), expected_stored_conf_val,
type(actual_stored_conf_val), actual_stored_conf_val))
def test_configure_default_sets_protocol_scheme_to_http(self):
mock_config_file = self.patch('app.util.conf.base_config_loader.ConfigFile').return_value
config = Configuration.singleton()
config_loader = SlaveConfigLoader()
config_loader.configure_defaults(config)
key = 'protocol_scheme'
expected_stored_protocol_scheme_value = 'http'
actual_stored_protocol_scheme_value = Configuration[key]
self.assertEqual(expected_stored_protocol_scheme_value, actual_stored_protocol_scheme_value,
'The configuration value for the key "{}" was expected to be {}:{}, but was {}:{}.'.format(
key, type(expected_stored_protocol_scheme_value), expected_stored_protocol_scheme_value,
type(actual_stored_protocol_scheme_value), actual_stored_protocol_scheme_value))
def test_configure_default_sets_protocol_scheme_to_http(self):
mock_config_file = self.patch(
'app.util.conf.base_config_loader.ConfigFile').return_value
config = Configuration.singleton()
config_loader = MasterConfigLoader()
config_loader.configure_defaults(config)
key = 'protocol_scheme'
expected_stored_protocol_scheme_value = 'http'
actual_stored_protocol_scheme_value = Configuration[key]
self.assertEqual(
expected_stored_protocol_scheme_value,
actual_stored_protocol_scheme_value,
'The configuration value for the key "{}" was expected to be {}:{}, but was {}:{}.'
.format(key, type(expected_stored_protocol_scheme_value),
expected_stored_protocol_scheme_value,
type(actual_stored_protocol_scheme_value),
actual_stored_protocol_scheme_value))
def test_configure_postload_sets_protocol_scheme_to_https(self):
mock_config_file = self.patch('app.util.conf.base_config_loader.ConfigFile').return_value
mock_config_file.read_config_from_disk.return_value = {'general': {'https_cert_file': '/path/to/cert',
'https_key_file': '/path/to/key'},
'slave': {}
}
config = Configuration.singleton()
config_loader = SlaveConfigLoader()
config_loader.configure_defaults(config)
config_loader.load_from_config_file(config, config_filename='fake_filename')
config_loader.configure_postload(config)
key = 'protocol_scheme'
expected_stored_protocol_scheme_value = 'https'
actual_stored_protocol_scheme_value = Configuration[key]
self.assertEqual(expected_stored_protocol_scheme_value, actual_stored_protocol_scheme_value,
'The configuration value for the key "{}" was expected to be {}:{}, but was {}:{}.'.format(
key, type(expected_stored_protocol_scheme_value), expected_stored_protocol_scheme_value,
type(actual_stored_protocol_scheme_value), actual_stored_protocol_scheme_value))
def test_configure_postload_sets_protocol_scheme_to_https(self):
mock_config_file = self.patch(
'app.util.conf.base_config_loader.ConfigFile').return_value
mock_config_file.read_config_from_disk.return_value = {
'general': {
'https_cert_file': '/path/to/cert',
'https_key_file': '/path/to/key'
},
'master': {}
}
# Unpached the patched method in BaseUnitTestCase for only this test case
# load_from_config_file is needed to update the Configuration with above "https_cert_file" and
# "https_key_file" values
self.unpatch(
'app.util.conf.master_config_loader.MasterConfigLoader.load_from_config_file'
)
config = Configuration.singleton()
config_loader = MasterConfigLoader()
config_loader.configure_defaults(config)
config_loader.load_from_config_file(config,
config_filename='fake_filename')
config_loader.configure_postload(config)
key = 'protocol_scheme'
expected_stored_protocol_scheme_value = 'https'
actual_stored_protocol_scheme_value = Configuration[key]
self.assertEqual(
expected_stored_protocol_scheme_value,
actual_stored_protocol_scheme_value,
'The configuration value for the key "{}" was expected to be {}:{}, but was {}:{}.'
.format(key, type(expected_stored_protocol_scheme_value),
expected_stored_protocol_scheme_value,
type(actual_stored_protocol_scheme_value),
actual_stored_protocol_scheme_value))
def setUp(self):
super().setUp()
SlaveConfigLoader().configure_defaults(Configuration.singleton())
SlaveConfigLoader().configure_postload(Configuration.singleton())
def setUp(self):
super().setUp()
SlaveConfigLoader().configure_defaults(Configuration.singleton())
SlaveConfigLoader().configure_postload(Configuration.singleton())
def test_conf_values_can_be_set_via_set_method(self):
conf = Configuration.singleton()
for conf_key, conf_value in self._test_conf_values.items():
conf.set(conf_key, conf_value)
self._assert_conf_values_match_expected(self._test_conf_values)
def _reset_config(self):
Configuration.reset_singleton()
config = Configuration.singleton()
conf_loader = BaseConfigLoader()
conf_loader.configure_defaults(config)
conf_loader.configure_postload(config)
def test_conf_values_can_be_set_via_set_method(self):
conf = Configuration.singleton()
for conf_key, conf_value in self._test_conf_values.items():
conf.set(conf_key, conf_value)
self._assert_conf_values_match_expected(self._test_conf_values)