def test_should_filter_directory_with_custom_replacer (self):
def replacer (token, replacement):
return replacement.upper()
self.create_tmp_dir("VARIABLES.localhost")
self.create_tmp_file(("VARIABLES.localhost", "SPAM"), "spam")
self.create_tmp_file(("VARIABLES.localhost", "EGGS"), "eggs")
self.create_tmp_file(("VARIABLES.localhost", "HAM"), "ham")
self.create_tmp_dir("is24-config-localhost")
self.create_tmp_file(("is24-config-localhost", "motd"),
"Today we serve @@@SPAM@@@ and @@@EGGS@@@.")
self.create_tmp_dir(("is24-config-localhost", "tomorrow"))
self.create_tmp_file(("is24-config-localhost", "tomorrow", "motd"),
"Tomorrow we serve @@@HAM@@@ and @@@EGGS@@@.")
TokenReplacer.filter_directory(self.tmp_directory,
self.abspath("VARIABLES.localhost"),
replacer_function=replacer)
self.ensure_file_contents(("is24-config-localhost", "motd"),
"Today we serve SPAM and EGGS.")
self.ensure_file_contents(("is24-config-localhost", "tomorrow", "motd"),
"Tomorrow we serve HAM and EGGS.")
def test_should_not_filter_file_with_encoding_unknown_8bit(self, mock_get_size, mock_config):
mock_get_size.return_value = 10
mock_config.return_value = 20
mock_token_replacer = Mock(TokenReplacer)
mock_token_replacer._read_content_from_file.return_value = 'fake binary file content'
mock_token_replacer._get_file_encoding.return_value = 'unknown-8bit'
TokenReplacer.filter_file(mock_token_replacer, "binary.file")
self.assertEqual(0, mock_token_replacer._perform_filtering_on_file.call_count)
def test_should_filter_directory_with_var_in_var(self):
self.create_tmp_dir("VARIABLES.localhost")
self.create_tmp_file(("VARIABLES.localhost", "FOO"), "@@@BAR@@@")
self.create_tmp_file(("VARIABLES.localhost", "BAR"), "bar")
self.create_tmp_dir("is24-config-localhost")
self.create_tmp_file(("is24-config-localhost", "text"),
"Hello @@@FOO@@@ and @@@BAR@@@.")
TokenReplacer.filter_directory(self.tmp_directory,
self.abspath("VARIABLES.localhost"))
self.ensure_file_contents(("is24-config-localhost", "text"),
"Hello bar and bar.")
def test_should_not_filter_file_with_encoding_unknown_8bit(
self, mock_get_size, mock_config):
mock_get_size.return_value = 10
mock_config.return_value = 20
mock_token_replacer = Mock(TokenReplacer)
mock_token_replacer._read_content_from_file.return_value = 'fake binary file content'
mock_token_replacer._get_file_encoding.return_value = 'unknown-8bit'
TokenReplacer.filter_file(mock_token_replacer, "binary.file")
self.assertEqual(
0, mock_token_replacer._perform_filtering_on_file.call_count)
def test_should_return_value_of_token_when_content_has_surrounding_white_spaces(
self):
self.assertEquals(
"spam",
TokenReplacer({
"SPAM": " spam\n"
}).filter("@@@SPAM@@@"))
def test_should_replace_token_in_token(self):
self.assertEquals(
"foo",
TokenReplacer({
"FOO": "foo",
"BAR": "@@@FOO@@@"
}).filter("@@@BAR@@@"))
def _test_should_return_value_of_replaced_token_when_content_is_token_reference_and_token_contains_special_characters(
self):
self.assertEquals(
"#\\/@$%&",
TokenReplacer({
"SPAM": "#\\/@$%&"
}).filter("@@@SPAM@@@"))
def test_should_strip_whitespaces_and_newlines_from_token_files (self):
self.create_tmp_file("SPAM", " spam")
self.create_tmp_file("EGGS", "eggs\n")
token_replacer = TokenReplacer.from_directory(self.tmp_directory)
self.assertEquals({'SPAM': 'spam', 'EGGS': 'eggs'}, token_replacer.token_values)
def test_should_return_value_of_content_with_replaced_tokens_when_content_is_a_mixture_of_static_text_and_tokens(
self):
self.assertEquals(
"You should eat more spam and eggs.",
TokenReplacer({
"SPAM": "spam",
"EGGS": "eggs"
}).filter("You should eat more @@@SPAM@@@ and @@@EGGS@@@."))
def test_should_return_value_of_two_tokens_when_content_is_double_token(
self):
self.assertEquals(
"spameggs",
TokenReplacer({
"SPAM": "spam",
"EGGS": "eggs"
}).filter("@@@SPAM@@@@@@EGGS@@@"))
def test_should_return_token_replacer_for_existing_directory (self):
self.create_tmp_file("SPAM", "spam")
self.create_tmp_file("EGGS", "eggs")
token_replacer = TokenReplacer.from_directory(self.tmp_directory)
self.assertEquals({'SPAM': 'spam', 'EGGS': 'eggs'},
token_replacer.token_values)
def test_should_use_custom_replacer_function(self):
def custom_replacer_function(token, value):
return "<%s:%s>" % (token, value)
self.assertEquals(
"<spam:eggs>",
TokenReplacer({
"spam": "eggs"
}, custom_replacer_function).filter("@@@spam@@@"))
def _filter_tokens_in_config_viewer(self):
def configviewer_token_replacer (token, replacement):
filtered_replacement = replacement.rstrip()
return '<strong title="%s">%s</strong>' % (token, filtered_replacement)
token_replacer = TokenReplacer.filter_directory(self.config_viewer_host_dir, self.variables_dir, html_escape=True, replacer_function=configviewer_token_replacer)
tokens_unused = set(token_replacer.token_values.keys()) - token_replacer.token_used
path_to_unused_variables = os.path.join(self.config_viewer_host_dir, 'unused_variables.txt')
self._write_file(path_to_unused_variables, '\n'.join(sorted(tokens_unused)))
token_replacer.filter_file(path_to_unused_variables, html_escape=True)
def test_should_filter_directory (self):
self.create_tmp_dir("VARIABLES.localhost")
self.create_tmp_file(("VARIABLES.localhost", "SPAM"), "spam")
self.create_tmp_file(("VARIABLES.localhost", "EGGS"), "eggs")
self.create_tmp_file(("VARIABLES.localhost", "HAM"), "ham")
self.create_tmp_dir("is24-config-localhost")
self.create_tmp_file(("is24-config-localhost", "motd"),
"Today we serve @@@SPAM@@@ and @@@EGGS@@@.")
self.create_tmp_dir(("is24-config-localhost", "tomorrow"))
self.create_tmp_file(("is24-config-localhost", "tomorrow", "motd"),
"Tomorrow we serve @@@HAM@@@ and @@@EGGS@@@.")
TokenReplacer.filter_directory(self.tmp_directory,
self.abspath("VARIABLES.localhost"))
self.ensure_file_contents(("is24-config-localhost", "motd"),
"Today we serve spam and eggs.")
self.ensure_file_contents(("is24-config-localhost", "tomorrow", "motd"),
"Tomorrow we serve ham and eggs.")
def _filter_tokens_in_config_viewer(self):
def configviewer_token_replacer(token, replacement):
filtered_replacement = replacement.rstrip()
return '<strong title="%s">%s</strong>' % (token, filtered_replacement)
token_replacer = TokenReplacer.filter_directory(self.config_viewer_host_dir,
self.variables_dir,
html_escape=True,
replacer_function=configviewer_token_replacer,
thread_name=self.thread_name)
tokens_unused = set(token_replacer.token_values.keys()) - token_replacer.token_used
path_to_unused_variables = os.path.join(self.config_viewer_host_dir, 'unused_variables.txt')
self._write_file(path_to_unused_variables, '\n'.join(sorted(tokens_unused)))
token_replacer.filter_file(path_to_unused_variables, html_escape=True)
def test_should_filter_directory_with_unused_variables (self):
self.create_tmp_dir("VARIABLES.localhost")
self.create_tmp_file(("VARIABLES.localhost", "SPAM"), "spam")
self.create_tmp_file(("VARIABLES.localhost", "EGGS"), "eggs")
self.create_tmp_file(("VARIABLES.localhost", "UNUSED"), "unused variable")
self.create_tmp_dir("is24-config-localhost")
self.create_tmp_file(("is24-config-localhost", "motd"),
"Today we serve @@@SPAM@@@ and @@@EGGS@@@.")
replacer = TokenReplacer.filter_directory(self.tmp_directory,
self.abspath("VARIABLES.localhost"))
self.assertEqual(replacer.token_used, set(["SPAM","EGGS"]))
def _filter_tokens_in_rpm_sources(self):
TokenReplacer.filter_directory(self.host_config_dir, self.variables_dir, thread_name=self.thread_name)
def test_should_not_replace_token_if_content_contains_invalid_token_reference(
self):
self.assertEquals("@@@SPAM@@",
TokenReplacer({
"SPAM": "spam"
}).filter("@@@SPAM@@"))
def build(self):
LOGGER.info('%s: building configuration rpm(s) for host "%s"', self.thread_name, self.hostname)
self.logger.info("Building config rpm for host %s revision %s", self.hostname, self.revision)
if exists(self.host_config_dir):
raise ConfigDirAlreadyExistsException('ERROR: "%s" exists already whereas I should be creating it now.' % self.host_config_dir)
try:
mkdir(self.host_config_dir)
except Exception as exception:
raise CouldNotCreateConfigDirException('Could not create host config directory "%s".' % self.host_config_dir, exception)
overall_requires = []
overall_provides = []
overall_svn_paths = []
overall_exported = {}
for segment in OVERLAY_ORDER:
svn_paths, exported_paths, requires, provides = self._overlay_segment(segment)
overall_exported[segment] = exported_paths
overall_svn_paths += svn_paths
overall_requires += requires
overall_provides += provides
self.logger.debug("Overall_exported: %s", str(overall_exported))
self.logger.info("Overall_requires: %s", str(overall_requires))
self.logger.info("Overall_provides: %s", str(overall_provides))
self.logger.debug("Overall_svn_paths: %s", str(overall_svn_paths))
if not exists(self.variables_dir):
mkdir(self.variables_dir)
self._write_dependency_file(overall_requires, self.rpm_requires_path, collapse_duplicates=True)
self._write_dependency_file(overall_provides, self.rpm_provides_path, False)
self._write_file(os.path.join(self.variables_dir, 'REVISION'), self.revision)
rpm_name_variable_file = os.path.join(self.variables_dir, 'RPM_NAME')
self.is_a_group_rpm = exists(rpm_name_variable_file)
do_not_write_host_segment_variable = self.is_a_group_rpm
self._save_segment_variables(do_not_write_host_segment_variable)
if self.is_a_group_rpm:
try:
TokenReplacer.filter_directory(os.path.dirname(self.variables_dir), self.variables_dir, thread_name=self.thread_name, skip=False)
except Exception as e:
LOGGER.warning("Problem during preliminary filtering of variables for group {0}: {1}".format(self.hostname, e))
with open(rpm_name_variable_file) as f:
self.rpm_name = f.read().rstrip()
LOGGER.info('Host {0} will trigger group rpm build with name {1}'.format(self.hostname, self.rpm_name))
self.spec_file_path = os.path.join(self.host_config_dir, self.config_rpm_prefix + self.rpm_name + '.spec')
self._write_file(os.path.join(self.variables_dir, 'INSTALL_PROTECTION_DEPENDENCY'), '')
else:
self._write_file(rpm_name_variable_file, self.hostname)
self.rpm_name = self.hostname
self._write_file(os.path.join(self.variables_dir, 'INSTALL_PROTECTION_DEPENDENCY'), 'hostname-@@@HOST@@@')
repo_packages_regex = get_repo_packages_regex()
self._write_dependency_file(overall_requires, os.path.join(self.variables_dir, 'RPM_REQUIRES_REPOS'), filter_regex=repo_packages_regex)
self._write_dependency_file(overall_requires, os.path.join(self.variables_dir, 'RPM_REQUIRES_NON_REPOS'), filter_regex=repo_packages_regex, positive_filter=False)
self._export_spec_file()
self._save_log_entries_to_variable(overall_svn_paths)
self._save_overlaying_to_variable(overall_exported)
self._move_variables_out_of_rpm_dir()
self._save_file_list()
self._save_network_variables()
patch_info = self._generate_patch_info()
self._copy_files_for_config_viewer()
# write patch info into variable and config viewer
self._write_file(os.path.join(self.variables_dir, 'VARIABLES'), patch_info)
self._write_file(os.path.join(self.config_viewer_host_dir, self.hostname + '.variables'), patch_info)
self._filter_tokens_in_rpm_sources()
if not is_config_viewer_only_enabled():
self._build_rpm_using_rpmbuild()
LOGGER.debug('%s: writing configviewer data for host "%s"', self.thread_name, self.hostname)
self._filter_tokens_in_config_viewer()
self._write_revision_file_for_config_viewer()
self._write_overlaying_for_config_viewer(overall_exported)
self._remove_logger_handlers()
self._clean_up()
return self._find_rpms()
def test_should_return_unmodified_content_when_content_does_not_contain_token(
self):
self.assertEquals("spam", TokenReplacer().filter("spam"))
def test_should_raise_exception_when_token_to_filter_is_missing(self):
self.assertRaises(MissingTokenException,
TokenReplacer().filter, "@@@NOT_FOUND@@@")
def test_should_return_value_of_token_when_content_is_single_token(self):
self.assertEquals("spam",
TokenReplacer({
"SPAM": "spam"
}).filter("@@@SPAM@@@"))
def _filter_tokens_in_rpm_sources(self):
TokenReplacer.filter_directory(self.host_config_dir, self.variables_dir)