def _create_logger(self):
logger = logging.getLogger(self.hostname)
self.handler = logging.FileHandler(os.path.join(self.work_dir, self.hostname + '.output'))
self.handler.setFormatter(logging.Formatter(self.LOG_FORMAT, self.DATE_FORMAT))
self.handler.setLevel(config.get('log_level', logging.INFO))
logger.addHandler(self.handler)
self.error_handler = logging.FileHandler(os.path.join(self.work_dir, self.hostname + '.error'))
self.error_handler.setFormatter(logging.Formatter(self.LOG_FORMAT, self.DATE_FORMAT))
self.error_handler.setLevel(logging.ERROR)
logger.addHandler(self.error_handler)
logger.setLevel(config.get('log_level', logging.INFO))
if self.error_logging_handler:
logger.addHandler(self.error_logging_handler)
return logger
def assertRpm(self, hostname, rpms, requires = None, provides = None, files=None):
path = None
for rpm_name in rpms:
name = os.path.basename(rpm_name)
if name.startswith(config_dev.get('config_rpm_prefix') + hostname) and 'noarch' in name and not 'repos' in name:
path = rpm_name
break
if not path:
raise Exception("Did not find host '%s' in %s." % (hostname, str(rpms)))
self.assertTrue(os.path.exists(path), "Could not find file %s ." % path)
ts = rpm.TransactionSet()
ts.setVSFlags((rpm._RPMVSF_NOSIGNATURES|rpm._RPMVSF_NODIGESTS))
f = open(path, 'r')
try:
hdr = ts.hdrFromFdno(f)
del ts
self.assertRequires(hdr, hostname, requires)
self.assertProvides(hdr, hostname, provides)
finally:
f.close()
extract_path = self.extractRpmFiles(path, hostname)
self.assertFiles(files, extract_path)
def _export_spec_file(self):
svn_service = self.svn_service_queue.get()
try:
svn_service.export(config.get('path_to_spec_file'), self.spec_file_path, self.revision)
finally:
self.svn_service_queue.put(svn_service)
self.svn_service_queue.task_done()
def get_config_viewer_host_dir(cls, hostname, temp=False):
path = os.path.join(config.get('config_viewer_dir'), 'hosts', hostname)
if temp:
path += '.new'
return path
def extractRpmFiles(self, path, hostname):
extract_path = os.path.join(config_dev.get('temp_dir'), hostname + '.extract')
os.mkdir(extract_path)
p = subprocess.Popen('rpm2cpio ' + os.path.abspath(path) + ' | cpio -idmv', shell=True, cwd=extract_path)
p.communicate()
return extract_path
def setUp(self):
temporary_directory = config.get(KEY_TEMPORARY_DIRECTORY)
if exists(temporary_directory):
rmtree(temporary_directory)
self.temporary_directory = temporary_directory
self.create_svn_repo()
def test_chunked_uploads(self):
old_config = config.get('rpm_upload_cmd')
target_file = os.path.abspath(os.path.join(config.get('temp_dir'), 'upload.txt'))
if os.path.exists(target_file):
os.remove(target_file)
cmd_file = os.path.abspath(os.path.join(config.get('temp_dir'), 'upload.sh'))
with open(cmd_file, 'w') as f:
f.write('#!/bin/bash\ndest=$1 ; shift ; echo "${#@} $@" >> "$dest"')
os.chmod(cmd_file, 0755)
cmd = '%s %s' % (cmd_file, target_file)
config.setvalue('rpm_upload_cmd',cmd)
try:
ConfigRpmMaker(None, None)._upload_rpms(['a' for x in range(25)])
finally:
config.setvalue('rpm_upload_cmd',old_config)
self.assertTrue(os.path.exists(target_file))
with open(target_file) as f:
self.assertEqual(f.read(), '10 a a a a a a a a a a\n10 a a a a a a a a a a\n5 a a a a a\n')
def create_svn_repo(self):
self.repo_dir = os.path.abspath(os.path.join(config.get('temp_dir'), 'svn_repo'))
if os.path.exists(self.repo_dir):
shutil.rmtree(self.repo_dir)
os.makedirs(self.repo_dir)
if subprocess.call('svnadmin create %s' % self.repo_dir, shell=True):
raise Exception('Could not create svn repo.')
self.repo_url = 'file://%s' % self.repo_dir
if subprocess.call('svn import -q -m import testdata/svn_repo %s' % self.repo_url, shell=True):
raise Exception('Could not import test data.')
if subprocess.call('svn import -q -m import testdata/index.html %s/config/typ/web/data/index.html' % self.repo_url, shell=True):
raise Exception('Could not import test data.')
def resolve(self, hostname):
dns_searchlist = config.get('custom_dns_searchlist')
if dns_searchlist:
for dns_entry in dns_searchlist:
try:
return self._resolve(hostname + '.' + dns_entry)
except Exception:
pass
else:
try:
return self._resolve(hostname)
except Exception:
pass
if not config.get('allow_unknown_hosts'):
raise Exception("Could not lookup '%s' with 'getent hosts'" % hostname)
ip = "127.0.0.1"
fqdn = "localhost.localdomain"
aliases = ""
return ip, fqdn, aliases
def _save_network_variables(self):
ip, fqdn, aliases = HostResolver().resolve(self.hostname)
if not ip:
if config.get('allow_unknown_hosts'):
ip = "127.0.0.1"
fqdn = "localhost.localdomain"
aliases = ""
else:
raise Exception("Could not lookup '%s' with 'getent hosts'" % self.hostname)
self._write_file(os.path.join(self.variables_dir, 'IP'), ip)
self._write_file(os.path.join(self.variables_dir, 'FQDN'), fqdn)
self._write_file(os.path.join(self.variables_dir, 'ALIASES'), aliases)
def resolve(self, hostname):
dns_searchlist = config.get('custom_dns_searchlist')
if dns_searchlist:
for dns_entry in dns_searchlist:
try:
return self._resolve(hostname + '.' + dns_entry)
except Exception:
pass
else:
try:
return self._resolve(hostname)
except Exception:
pass
return None, None, None
def __init__(self, hostname, revision, work_dir, svn_service_queue, error_logging_handler = None):
self.hostname = hostname
self.revision = revision
self.work_dir = work_dir
self.error_logging_handler = error_logging_handler
self.logger = self._create_logger()
self.svn_service_queue = svn_service_queue
self.config_rpm_prefix = config.get('config_rpm_prefix')
self.host_config_dir = os.path.join(self.work_dir, self.config_rpm_prefix + self.hostname)
self.variables_dir = os.path.join(self.host_config_dir, 'VARIABLES')
self.rpm_requires_path = os.path.join(self.variables_dir, 'RPM_REQUIRES')
self.rpm_provides_path = os.path.join(self.variables_dir, 'RPM_PROVIDES')
self.spec_file_path = os.path.join(self.host_config_dir, self.config_rpm_prefix + self.hostname + '.spec')
self.config_viewer_host_dir = HostRpmBuilder.get_config_viewer_host_dir(hostname, True)
self.rpm_build_dir = os.path.join(self.work_dir, 'rpmbuild')
def mainMethod(args=sys.argv[1:]):
try:
if len(args) != 2:
raise CliException("You need to provide 2 parameters (repo dir, revision).\nArguments were %s " % str(args))
if not (args[1].isdigit() and int(args[1]) >= 0):
raise CliException("Revision must be a positive integer.\nGiven revision was '%s'" % args[1])
# first use case is post-commit hook. repo dir can be used as file:/// SVN URL
svn_service = SvnService(
base_url = 'file://' + args[0],
path_to_config = config.get('svn_path_to_config')
)
ConfigRpmMaker(revision=args[1], svn_service=svn_service).build()
except (BaseConfigRpmMakerException) as e:
print "See the error log for details."
sys.exit(1)
except Exception:
traceback.print_exc(5)
sys.exit(2)
def extractRpmFiles(self, path, hostname):
extract_path = os.path.join(config_dev.get('temp_dir'), hostname + '.extract')
os.mkdir(extract_path)
command_with_arguments = 'rpm2cpio ' + os.path.abspath(path) + ' | cpio -idmv'
process = subprocess.Popen(command_with_arguments,
shell=True,
cwd=extract_path,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
if process.returncode != 0:
message = EXECUTION_ERROR_MESSAGE.format(command_with_arguments=command_with_arguments,
error_code=process.returncode,
stdout=stdout or "",
stderr=stderr or "")
raise IntegrationTestException(message)
return extract_path
def mainMethod(args=sys.argv[1:]):
try:
if len(args) != 2:
raise CliException("You need to provide 2 parameters (repo dir, revision).\nArguments were %s " % str(args))
if not (args[1].isdigit() and int(args[1]) >= 0):
raise CliException("Revision must be a positive integer.\nGiven revision was '%s'" % args[1])
# first use case is post-commit hook. repo dir can be used as file:/// SVN URL
svn_service = SvnService(base_url="file://" + args[0], path_to_config=config.get("svn_path_to_config"))
config_rpm_maker = ConfigRpmMaker(revision=args[1], svn_service=svn_service)
result = config_rpm_maker.build()
ConfigRpmUploader().upload_rpms(result.rpms)
ConfigViewerAdapter().move_configviewer_dirs_to_final_destination(result.affectedHosts)
config_rpm_maker.clean_up()
except (BaseConfigRpmMakerException) as e:
print "See the error log for details."
sys.exit(1)
except Exception:
traceback.print_exc(5)
sys.exit(2)
def _create_logger(self):
log_level = config.get(KEY_LOG_LEVEL)
formatter = Formatter(config.LOG_FILE_FORMAT, config.LOG_FILE_DATE_FORMAT)
self.handler = FileHandler(os.path.join(self.work_dir, self.hostname + '.output'))
self.handler.setFormatter(formatter)
self.handler.setLevel(log_level)
self.error_handler = FileHandler(os.path.join(self.work_dir, self.hostname + '.error'))
self.error_handler.setFormatter(formatter)
self.error_handler.setLevel(ERROR)
logger = getLogger(self.hostname)
logger.addHandler(self.handler)
logger.addHandler(self.error_handler)
logger.setLevel(log_level)
if self.error_logging_handler:
logger.addHandler(self.error_logging_handler)
return logger
def filter_file (self, filename, html_escape=False):
__pychecker__ = "missingattrs=token"
try:
self.file_size_limit = config.get('max_file_size', 100 * 1024)
if os.path.getsize(filename) > self.file_size_limit:
raise Exception("FileTooFatException : %s\n\t(size is %s bytes, limit is %s bytes)"%(os.path.basename(filename),os.path.getsize(filename), self.file_size_limit))
with open(filename, "r") as input_file:
file_content = input_file.read()
file_encoding = self._get_file_encoding(file_content)
if file_encoding and file_encoding != 'binary':
file_content = file_content.decode(file_encoding)
if html_escape:
file_content = self.html_escape_function(os.path.basename(filename), file_content)
file_content_filtered = self.filter(file_content)
with open(filename, "w") as output_file:
output_file.write(file_content_filtered.encode(file_encoding))
except MissingTokenException as exception:
raise MissingTokenException(exception.token, filename)
except Exception as e:
raise CannotFilterFileException('Cannot filter file %s.\n%s'%(os.path.basename(filename),str(e)))
def _given_config_rpm_maker(self, keep_work_dir = False):
self._cleanup_temp_dir()
self.create_svn_repo()
svn_service = SvnService(base_url=self.repo_url, username=None, password=None, path_to_config=config.get('svn_path_to_config'))
if keep_work_dir:
os.environ['KEEPWORKDIR'] = '1'
elif os.environ.has_key('KEEPWORKDIR'):
del os.environ['KEEPWORKDIR']
return ConfigRpmMaker('2', svn_service)
def _cleanup_temp_dir(self):
temp_dir = config_dev.get('temp_dir')
if temp_dir:
if os.path.exists(temp_dir):
shutil.rmtree(temp_dir)
os.makedirs(temp_dir)
def test_get_hosts(self):
service = SvnService(self.repo_url, None, None, path_to_config = config.get('svn_path_to_config'))
self.assertEqual(['berweb01', 'devweb01', 'tuvweb01'], service.get_hosts(2))
def build(self):
self.logger.info("Building config rpm for host %s revision %s", self.hostname, self.revision)
if os.path.exists(self.host_config_dir):
raise Exception("ERROR: '%s' exists already whereas I should be creating it now." % self.host_config_dir)
try:
os.mkdir(self.host_config_dir)
except Exception as e:
raise CouldNotCreateConfigDirException("Could not create host config directory '%s' : %s" % self.host_config_dir, e)
overall_requires = []
overall_provides = []
overall_svn_paths = []
overall_exported = {}
for segment in OVERLAY_ORDER:
svn_paths, exported_paths, requires, provides = self._overlay_segment(segment)
overall_exported[segment] = exported_paths
overall_svn_paths += svn_paths
overall_requires += requires
overall_provides += provides
self.logger.debug("Overall_exported: %s", str(overall_exported))
self.logger.info("Overall_requires: %s", str(overall_requires))
self.logger.info("Overall_provides: %s", str(overall_provides))
self.logger.debug("Overall_svn_paths: %s", str(overall_svn_paths))
if not os.path.exists(self.variables_dir):
os.mkdir(self.variables_dir)
self._write_dependency_file(overall_requires, self.rpm_requires_path, collapse_duplicates=True)
self._write_dependency_file(overall_provides, self.rpm_provides_path, False)
self._write_file(os.path.join(self.variables_dir, 'REVISION'), self.revision)
repo_packages_regex = config.get('repo_packages_regex')
if repo_packages_regex:
self._write_dependency_file(overall_requires, os.path.join(self.variables_dir, 'RPM_REQUIRES_REPOS'), filter_regex=repo_packages_regex)
self._write_dependency_file(overall_requires, os.path.join(self.variables_dir, 'RPM_REQUIRES_NON_REPOS'), filter_regex=repo_packages_regex, positive_filter=False)
self._export_spec_file()
self._save_log_entries_to_variable(overall_svn_paths)
self._save_overlaying_to_variable(overall_exported)
self._move_variables_out_of_rpm_dir()
self._save_file_list()
self._save_segment_variables()
self._save_network_variables()
patch_info = self._generate_patch_info()
self._copy_files_for_config_viewer()
# write patch info into variable and config viewer
self._write_file(os.path.join(self.variables_dir, 'VARIABLES'), patch_info)
self._write_file(os.path.join(self.config_viewer_host_dir, self.hostname + '.variables'), patch_info)
self._filter_tokens_in_rpm_sources()
self._build_rpm()
self._filter_tokens_in_config_viewer()
self._write_revision_file_for_config_viewer()
self._write_overlaying_for_config_viewer(overall_exported)
self._remove_logger_handlers()
return self._find_rpms()
def test_log(self):
service = SvnService(self.repo_url, None, None, path_to_config = config.get('svn_path_to_config'))
logs = service.log('', 2, 5)
self.assertEqual(1, len(logs))
def test_export(self):
service = SvnService(self.repo_url, None, None, path_to_config = config.get('svn_path_to_config'))
self.assertEqual([('host/berweb01', 'VARIABLES'), ('host/berweb01', 'VARIABLES/RPM_REQUIRES')], service.export('host/berweb01', 'target/tmp/test', 2))
def create_svn_service_queue(self):
svn_service = SvnService(base_url=self.repo_url, username=None, password=None,
path_to_config=config.get(KEY_SVN_PATH_TO_CONFIG))
svn_service_queue = Queue()
svn_service_queue.put(svn_service)
return svn_service_queue
def test_get_change_set(self):
service = SvnService(self.repo_url, None, None, path_to_config = config.get('svn_path_to_config'))
self.assertEqual(['typ/web/data/index.html'], service.get_change_set(2))
def log_execution_time_summaries(logging_function):
logging_function('Execution times summary (keep in mind thread_count was set to %s):', get(KEY_THREAD_COUNT))
for function_name in sorted(_summary.keys()):
summary_of_function = _summary[function_name]
rounded_elapsed_time = round_to_two_decimals_after_dot(summary_of_function[0])
average_time = round_to_two_decimals_after_dot(summary_of_function[0] / summary_of_function[1])
logging_function(' %3s times with average %5ss = sum %5ss : %s',
summary_of_function[1], average_time, rounded_elapsed_time, function_name)
from config_rpm_maker.configRpmMaker import ConfigRpmMaker
from config_rpm_maker.configRpmUploader import ConfigRpmUploader
from config_rpm_maker.configViewerAdapter import ConfigViewerAdapter
from config_rpm_maker.svn import SvnService
from config_rpm_maker.exceptions import BaseConfigRpmMakerException
from config_rpm_maker import config
import logging
import traceback
import sys
logging.basicConfig(format="%(asctime)s %(levelname)5s [%(name)s] - %(message)s", level=config.get("log_level", "INFO"))
class CliException(BaseConfigRpmMakerException):
error_info = "Command Line Error:\n"
def mainMethod(args=sys.argv[1:]):
try:
if len(args) != 2:
raise CliException("You need to provide 2 parameters (repo dir, revision).\nArguments were %s " % str(args))
if not (args[1].isdigit() and int(args[1]) >= 0):
raise CliException("Revision must be a positive integer.\nGiven revision was '%s'" % args[1])
# first use case is post-commit hook. repo dir can be used as file:/// SVN URL
svn_service = SvnService(base_url="file://" + args[0], path_to_config=config.get("svn_path_to_config"))
config_rpm_maker = ConfigRpmMaker(revision=args[1], svn_service=svn_service)
result = config_rpm_maker.build()
from config_rpm_maker.configRpmMaker import ConfigRpmMaker
from config_rpm_maker.svn import SvnService
from config_rpm_maker.exceptions import BaseConfigRpmMakerException
from config_rpm_maker import config
import logging
import traceback
import sys
logging.basicConfig(
format="%(asctime)s %(levelname)5s [%(name)s] - %(message)s",
level=config.get('log_level', 'INFO'),
)
class CliException(BaseConfigRpmMakerException):
error_info = "Command Line Error:\n"
def mainMethod(args=sys.argv[1:]):
try:
if len(args) != 2:
raise CliException("You need to provide 2 parameters (repo dir, revision).\nArguments were %s " % str(args))
if not (args[1].isdigit() and int(args[1]) >= 0):
raise CliException("Revision must be a positive integer.\nGiven revision was '%s'" % args[1])
# first use case is post-commit hook. repo dir can be used as file:/// SVN URL
svn_service = SvnService(