def test_should_filter_directory_with_custom_replacer (self):
def replacer (token, replacement):
return replacement.upper()
self.create_tmp_dir("VARIABLES.localhost")
self.create_tmp_file(("VARIABLES.localhost", "SPAM"), "spam")
self.create_tmp_file(("VARIABLES.localhost", "EGGS"), "eggs")
self.create_tmp_file(("VARIABLES.localhost", "HAM"), "ham")
self.create_tmp_dir("is24-config-localhost")
self.create_tmp_file(("is24-config-localhost", "motd"),
"Today we serve @@@SPAM@@@ and @@@EGGS@@@.")
self.create_tmp_dir(("is24-config-localhost", "tomorrow"))
self.create_tmp_file(("is24-config-localhost", "tomorrow", "motd"),
"Tomorrow we serve @@@HAM@@@ and @@@EGGS@@@.")
TokenReplacer.filter_directory(self.tmp_directory,
self.abspath("VARIABLES.localhost"),
replacer_function=replacer)
self.ensure_file_contents(("is24-config-localhost", "motd"),
"Today we serve SPAM and EGGS.")
self.ensure_file_contents(("is24-config-localhost", "tomorrow", "motd"),
"Tomorrow we serve HAM and EGGS.")
def test_should_filter_directory_with_var_in_var(self):
self.create_tmp_dir("VARIABLES.localhost")
self.create_tmp_file(("VARIABLES.localhost", "FOO"), "@@@BAR@@@")
self.create_tmp_file(("VARIABLES.localhost", "BAR"), "bar")
self.create_tmp_dir("is24-config-localhost")
self.create_tmp_file(("is24-config-localhost", "text"),
"Hello @@@FOO@@@ and @@@BAR@@@.")
TokenReplacer.filter_directory(self.tmp_directory,
self.abspath("VARIABLES.localhost"))
self.ensure_file_contents(("is24-config-localhost", "text"),
"Hello bar and bar.")
def _filter_tokens_in_config_viewer(self):
def configviewer_token_replacer (token, replacement):
filtered_replacement = replacement.rstrip()
return '<strong title="%s">%s</strong>' % (token, filtered_replacement)
token_replacer = TokenReplacer.filter_directory(self.config_viewer_host_dir, self.variables_dir, html_escape=True, replacer_function=configviewer_token_replacer)
tokens_unused = set(token_replacer.token_values.keys()) - token_replacer.token_used
path_to_unused_variables = os.path.join(self.config_viewer_host_dir, 'unused_variables.txt')
self._write_file(path_to_unused_variables, '\n'.join(sorted(tokens_unused)))
token_replacer.filter_file(path_to_unused_variables, html_escape=True)
def test_should_filter_directory (self):
self.create_tmp_dir("VARIABLES.localhost")
self.create_tmp_file(("VARIABLES.localhost", "SPAM"), "spam")
self.create_tmp_file(("VARIABLES.localhost", "EGGS"), "eggs")
self.create_tmp_file(("VARIABLES.localhost", "HAM"), "ham")
self.create_tmp_dir("is24-config-localhost")
self.create_tmp_file(("is24-config-localhost", "motd"),
"Today we serve @@@SPAM@@@ and @@@EGGS@@@.")
self.create_tmp_dir(("is24-config-localhost", "tomorrow"))
self.create_tmp_file(("is24-config-localhost", "tomorrow", "motd"),
"Tomorrow we serve @@@HAM@@@ and @@@EGGS@@@.")
TokenReplacer.filter_directory(self.tmp_directory,
self.abspath("VARIABLES.localhost"))
self.ensure_file_contents(("is24-config-localhost", "motd"),
"Today we serve spam and eggs.")
self.ensure_file_contents(("is24-config-localhost", "tomorrow", "motd"),
"Tomorrow we serve ham and eggs.")
def test_should_filter_directory_with_unused_variables (self):
self.create_tmp_dir("VARIABLES.localhost")
self.create_tmp_file(("VARIABLES.localhost", "SPAM"), "spam")
self.create_tmp_file(("VARIABLES.localhost", "EGGS"), "eggs")
self.create_tmp_file(("VARIABLES.localhost", "UNUSED"), "unused variable")
self.create_tmp_dir("is24-config-localhost")
self.create_tmp_file(("is24-config-localhost", "motd"),
"Today we serve @@@SPAM@@@ and @@@EGGS@@@.")
replacer = TokenReplacer.filter_directory(self.tmp_directory,
self.abspath("VARIABLES.localhost"))
self.assertEqual(replacer.token_used, set(["SPAM","EGGS"]))
def _filter_tokens_in_config_viewer(self):
def configviewer_token_replacer(token, replacement):
filtered_replacement = replacement.rstrip()
return '<strong title="%s">%s</strong>' % (token, filtered_replacement)
token_replacer = TokenReplacer.filter_directory(self.config_viewer_host_dir,
self.variables_dir,
html_escape=True,
replacer_function=configviewer_token_replacer,
thread_name=self.thread_name)
tokens_unused = set(token_replacer.token_values.keys()) - token_replacer.token_used
path_to_unused_variables = os.path.join(self.config_viewer_host_dir, 'unused_variables.txt')
self._write_file(path_to_unused_variables, '\n'.join(sorted(tokens_unused)))
token_replacer.filter_file(path_to_unused_variables, html_escape=True)
def build(self):
LOGGER.info('%s: building configuration rpm(s) for host "%s"', self.thread_name, self.hostname)
self.logger.info("Building config rpm for host %s revision %s", self.hostname, self.revision)
if exists(self.host_config_dir):
raise ConfigDirAlreadyExistsException('ERROR: "%s" exists already whereas I should be creating it now.' % self.host_config_dir)
try:
mkdir(self.host_config_dir)
except Exception as exception:
raise CouldNotCreateConfigDirException('Could not create host config directory "%s".' % self.host_config_dir, exception)
overall_requires = []
overall_provides = []
overall_svn_paths = []
overall_exported = {}
for segment in OVERLAY_ORDER:
svn_paths, exported_paths, requires, provides = self._overlay_segment(segment)
overall_exported[segment] = exported_paths
overall_svn_paths += svn_paths
overall_requires += requires
overall_provides += provides
self.logger.debug("Overall_exported: %s", str(overall_exported))
self.logger.info("Overall_requires: %s", str(overall_requires))
self.logger.info("Overall_provides: %s", str(overall_provides))
self.logger.debug("Overall_svn_paths: %s", str(overall_svn_paths))
if not exists(self.variables_dir):
mkdir(self.variables_dir)
self._write_dependency_file(overall_requires, self.rpm_requires_path, collapse_duplicates=True)
self._write_dependency_file(overall_provides, self.rpm_provides_path, False)
self._write_file(os.path.join(self.variables_dir, 'REVISION'), self.revision)
rpm_name_variable_file = os.path.join(self.variables_dir, 'RPM_NAME')
self.is_a_group_rpm = exists(rpm_name_variable_file)
do_not_write_host_segment_variable = self.is_a_group_rpm
self._save_segment_variables(do_not_write_host_segment_variable)
if self.is_a_group_rpm:
try:
TokenReplacer.filter_directory(os.path.dirname(self.variables_dir), self.variables_dir, thread_name=self.thread_name, skip=False)
except Exception as e:
LOGGER.warning("Problem during preliminary filtering of variables for group {0}: {1}".format(self.hostname, e))
with open(rpm_name_variable_file) as f:
self.rpm_name = f.read().rstrip()
LOGGER.info('Host {0} will trigger group rpm build with name {1}'.format(self.hostname, self.rpm_name))
self.spec_file_path = os.path.join(self.host_config_dir, self.config_rpm_prefix + self.rpm_name + '.spec')
self._write_file(os.path.join(self.variables_dir, 'INSTALL_PROTECTION_DEPENDENCY'), '')
else:
self._write_file(rpm_name_variable_file, self.hostname)
self.rpm_name = self.hostname
self._write_file(os.path.join(self.variables_dir, 'INSTALL_PROTECTION_DEPENDENCY'), 'hostname-@@@HOST@@@')
repo_packages_regex = get_repo_packages_regex()
self._write_dependency_file(overall_requires, os.path.join(self.variables_dir, 'RPM_REQUIRES_REPOS'), filter_regex=repo_packages_regex)
self._write_dependency_file(overall_requires, os.path.join(self.variables_dir, 'RPM_REQUIRES_NON_REPOS'), filter_regex=repo_packages_regex, positive_filter=False)
self._export_spec_file()
self._save_log_entries_to_variable(overall_svn_paths)
self._save_overlaying_to_variable(overall_exported)
self._move_variables_out_of_rpm_dir()
self._save_file_list()
self._save_network_variables()
patch_info = self._generate_patch_info()
self._copy_files_for_config_viewer()
# write patch info into variable and config viewer
self._write_file(os.path.join(self.variables_dir, 'VARIABLES'), patch_info)
self._write_file(os.path.join(self.config_viewer_host_dir, self.hostname + '.variables'), patch_info)
self._filter_tokens_in_rpm_sources()
if not is_config_viewer_only_enabled():
self._build_rpm_using_rpmbuild()
LOGGER.debug('%s: writing configviewer data for host "%s"', self.thread_name, self.hostname)
self._filter_tokens_in_config_viewer()
self._write_revision_file_for_config_viewer()
self._write_overlaying_for_config_viewer(overall_exported)
self._remove_logger_handlers()
self._clean_up()
return self._find_rpms()
def _filter_tokens_in_rpm_sources(self):
TokenReplacer.filter_directory(self.host_config_dir, self.variables_dir, thread_name=self.thread_name)
def _filter_tokens_in_rpm_sources(self):
TokenReplacer.filter_directory(self.host_config_dir, self.variables_dir)
