def _create_logger(self):
     logger = logging.getLogger(self.hostname)
     self.handler = logging.FileHandler(os.path.join(self.work_dir, self.hostname + '.output'))
     self.handler.setFormatter(logging.Formatter(self.LOG_FORMAT, self.DATE_FORMAT))
     self.handler.setLevel(config.get('log_level', logging.INFO))
     logger.addHandler(self.handler)
     self.error_handler = logging.FileHandler(os.path.join(self.work_dir, self.hostname + '.error'))
     self.error_handler.setFormatter(logging.Formatter(self.LOG_FORMAT, self.DATE_FORMAT))
     self.error_handler.setLevel(logging.ERROR)
     logger.addHandler(self.error_handler)
     logger.setLevel(config.get('log_level', logging.INFO))
     if self.error_logging_handler:
         logger.addHandler(self.error_logging_handler)
     return logger
    def assertRpm(self, hostname, rpms, requires = None, provides = None, files=None):
        path = None
        for rpm_name in rpms:
            name = os.path.basename(rpm_name)
            if name.startswith(config_dev.get('config_rpm_prefix') + hostname) and 'noarch' in name and not 'repos' in name:
                path = rpm_name
                break

        if not path:
            raise Exception("Did not find host '%s' in %s." % (hostname, str(rpms)))

        self.assertTrue(os.path.exists(path), "Could not find file %s ." % path)
        ts = rpm.TransactionSet()
        ts.setVSFlags((rpm._RPMVSF_NOSIGNATURES|rpm._RPMVSF_NODIGESTS))
        f = open(path, 'r')
        try:
            hdr = ts.hdrFromFdno(f)
            del ts
            self.assertRequires(hdr, hostname, requires)
            self.assertProvides(hdr, hostname, provides)
        finally:
            f.close()

        extract_path = self.extractRpmFiles(path, hostname)
        self.assertFiles(files, extract_path)
 def _export_spec_file(self):
     svn_service = self.svn_service_queue.get()
     try:
         svn_service.export(config.get('path_to_spec_file'), self.spec_file_path, self.revision)
     finally:
         self.svn_service_queue.put(svn_service)
         self.svn_service_queue.task_done()
    def get_config_viewer_host_dir(cls, hostname, temp=False):
        path = os.path.join(config.get('config_viewer_dir'), 'hosts', hostname)

        if temp:
            path += '.new'

        return path
    def extractRpmFiles(self, path, hostname):
        extract_path = os.path.join(config_dev.get('temp_dir'), hostname + '.extract')
        os.mkdir(extract_path)

        p = subprocess.Popen('rpm2cpio ' + os.path.abspath(path) + ' | cpio  -idmv', shell=True, cwd=extract_path)
        p.communicate()

        return extract_path
    def setUp(self):
        temporary_directory = config.get(KEY_TEMPORARY_DIRECTORY)

        if exists(temporary_directory):
            rmtree(temporary_directory)

        self.temporary_directory = temporary_directory

        self.create_svn_repo()
    def test_chunked_uploads(self):
        old_config = config.get('rpm_upload_cmd')
        target_file = os.path.abspath(os.path.join(config.get('temp_dir'), 'upload.txt'))
        if os.path.exists(target_file):
            os.remove(target_file)
        cmd_file = os.path.abspath(os.path.join(config.get('temp_dir'), 'upload.sh'))
        with open(cmd_file, 'w') as f:
            f.write('#!/bin/bash\ndest=$1 ; shift ; echo "${#@} $@" >> "$dest"')

        os.chmod(cmd_file, 0755)
        cmd = '%s %s' % (cmd_file, target_file)
        config.setvalue('rpm_upload_cmd',cmd)
        try:
            ConfigRpmMaker(None, None)._upload_rpms(['a' for x in range(25)])
        finally:
            config.setvalue('rpm_upload_cmd',old_config)

        self.assertTrue(os.path.exists(target_file))
        with open(target_file) as f:
            self.assertEqual(f.read(), '10 a a a a a a a a a a\n10 a a a a a a a a a a\n5 a a a a a\n')
 def create_svn_repo(self):
     self.repo_dir = os.path.abspath(os.path.join(config.get('temp_dir'), 'svn_repo'))
     if os.path.exists(self.repo_dir):
         shutil.rmtree(self.repo_dir)
     os.makedirs(self.repo_dir)
     if subprocess.call('svnadmin create %s' % self.repo_dir, shell=True):
         raise Exception('Could not create svn repo.')
     self.repo_url = 'file://%s' % self.repo_dir
     if subprocess.call('svn import -q -m import testdata/svn_repo %s' % self.repo_url, shell=True):
         raise Exception('Could not import test data.')
     if subprocess.call('svn import -q -m import testdata/index.html %s/config/typ/web/data/index.html' % self.repo_url, shell=True):
         raise Exception('Could not import test data.')
    def resolve(self, hostname):
        dns_searchlist = config.get('custom_dns_searchlist')

        if dns_searchlist:
            for dns_entry in dns_searchlist:
                try:
                    return self._resolve(hostname + '.' + dns_entry)
                except Exception:
                    pass
        else:
            try:
                return self._resolve(hostname)
            except Exception:
                pass

        if not config.get('allow_unknown_hosts'):
            raise Exception("Could not lookup '%s' with 'getent hosts'" % hostname)

        ip = "127.0.0.1"
        fqdn = "localhost.localdomain"
        aliases = ""
        return ip, fqdn, aliases
    def _save_network_variables(self):
        ip, fqdn, aliases = HostResolver().resolve(self.hostname)
        if not ip:
            if config.get('allow_unknown_hosts'):
                ip = "127.0.0.1"
                fqdn = "localhost.localdomain"
                aliases = ""
            else:
                raise Exception("Could not lookup '%s' with 'getent hosts'" % self.hostname)

        self._write_file(os.path.join(self.variables_dir, 'IP'), ip)
        self._write_file(os.path.join(self.variables_dir, 'FQDN'), fqdn)
        self._write_file(os.path.join(self.variables_dir, 'ALIASES'), aliases)
    def resolve(self, hostname):
        dns_searchlist = config.get('custom_dns_searchlist')
        if dns_searchlist:
            for dns_entry in dns_searchlist:
                try:
                    return self._resolve(hostname + '.' + dns_entry)
                except Exception:
                    pass
        else:
            try:
                return self._resolve(hostname)
            except Exception:
                pass

        return None, None, None
 def __init__(self, hostname, revision, work_dir, svn_service_queue, error_logging_handler = None):
     self.hostname = hostname
     self.revision = revision
     self.work_dir = work_dir
     self.error_logging_handler = error_logging_handler
     self.logger = self._create_logger()
     self.svn_service_queue = svn_service_queue
     self.config_rpm_prefix = config.get('config_rpm_prefix')
     self.host_config_dir = os.path.join(self.work_dir, self.config_rpm_prefix + self.hostname)
     self.variables_dir = os.path.join(self.host_config_dir, 'VARIABLES')
     self.rpm_requires_path = os.path.join(self.variables_dir, 'RPM_REQUIRES')
     self.rpm_provides_path = os.path.join(self.variables_dir, 'RPM_PROVIDES')
     self.spec_file_path = os.path.join(self.host_config_dir, self.config_rpm_prefix +  self.hostname + '.spec')
     self.config_viewer_host_dir = HostRpmBuilder.get_config_viewer_host_dir(hostname, True)
     self.rpm_build_dir = os.path.join(self.work_dir, 'rpmbuild')
def mainMethod(args=sys.argv[1:]):
    try:
        if len(args) != 2:
            raise CliException("You need to provide 2 parameters (repo dir, revision).\nArguments were %s " % str(args))
    
        if not (args[1].isdigit() and int(args[1]) >= 0):
            raise CliException("Revision must be a positive integer.\nGiven revision was '%s'" % args[1])
        
    # first use case is post-commit hook. repo dir can be used as file:/// SVN URL
        svn_service = SvnService(
            base_url = 'file://' + args[0],
            path_to_config = config.get('svn_path_to_config')
        )
        ConfigRpmMaker(revision=args[1], svn_service=svn_service).build()
    except (BaseConfigRpmMakerException) as e:
        print "See the error log for details."
        sys.exit(1)
    except Exception:
        traceback.print_exc(5)
        sys.exit(2)
    def extractRpmFiles(self, path, hostname):
        extract_path = os.path.join(config_dev.get('temp_dir'), hostname + '.extract')
        os.mkdir(extract_path)

        command_with_arguments = 'rpm2cpio ' + os.path.abspath(path) + ' | cpio  -idmv'
        process = subprocess.Popen(command_with_arguments,
                                   shell=True,
                                   cwd=extract_path,
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE)
        stdout, stderr = process.communicate()

        if process.returncode != 0:
            message = EXECUTION_ERROR_MESSAGE.format(command_with_arguments=command_with_arguments,
                                                     error_code=process.returncode,
                                                     stdout=stdout or "",
                                                     stderr=stderr or "")

            raise IntegrationTestException(message)

        return extract_path
def mainMethod(args=sys.argv[1:]):
    try:
        if len(args) != 2:
            raise CliException("You need to provide 2 parameters (repo dir, revision).\nArguments were %s " % str(args))

        if not (args[1].isdigit() and int(args[1]) >= 0):
            raise CliException("Revision must be a positive integer.\nGiven revision was '%s'" % args[1])

        # first use case is post-commit hook. repo dir can be used as file:/// SVN URL
        svn_service = SvnService(base_url="file://" + args[0], path_to_config=config.get("svn_path_to_config"))
        config_rpm_maker = ConfigRpmMaker(revision=args[1], svn_service=svn_service)
        result = config_rpm_maker.build()
        ConfigRpmUploader().upload_rpms(result.rpms)
        ConfigViewerAdapter().move_configviewer_dirs_to_final_destination(result.affectedHosts)
        config_rpm_maker.clean_up()
    except (BaseConfigRpmMakerException) as e:
        print "See the error log for details."
        sys.exit(1)
    except Exception:
        traceback.print_exc(5)
        sys.exit(2)
    def _create_logger(self):
        log_level = config.get(KEY_LOG_LEVEL)
        formatter = Formatter(config.LOG_FILE_FORMAT, config.LOG_FILE_DATE_FORMAT)

        self.handler = FileHandler(os.path.join(self.work_dir, self.hostname + '.output'))
        self.handler.setFormatter(formatter)
        self.handler.setLevel(log_level)

        self.error_handler = FileHandler(os.path.join(self.work_dir, self.hostname + '.error'))
        self.error_handler.setFormatter(formatter)
        self.error_handler.setLevel(ERROR)

        logger = getLogger(self.hostname)
        logger.addHandler(self.handler)
        logger.addHandler(self.error_handler)
        logger.setLevel(log_level)

        if self.error_logging_handler:
            logger.addHandler(self.error_logging_handler)

        return logger
    def filter_file (self, filename, html_escape=False):
        __pychecker__ = "missingattrs=token"
        try:
            self.file_size_limit = config.get('max_file_size', 100 * 1024)
            if os.path.getsize(filename) > self.file_size_limit:
              raise Exception("FileTooFatException : %s\n\t(size is %s bytes, limit is %s bytes)"%(os.path.basename(filename),os.path.getsize(filename), self.file_size_limit))

            with open(filename, "r") as input_file:
                file_content = input_file.read()

            file_encoding = self._get_file_encoding(file_content)
            if file_encoding and file_encoding != 'binary':
                file_content = file_content.decode(file_encoding)
                if html_escape:
                    file_content = self.html_escape_function(os.path.basename(filename), file_content)

                file_content_filtered = self.filter(file_content)
                with open(filename, "w") as output_file:
                    output_file.write(file_content_filtered.encode(file_encoding))
        except MissingTokenException as exception:
            raise MissingTokenException(exception.token, filename)
        except Exception as e:
            raise CannotFilterFileException('Cannot filter file %s.\n%s'%(os.path.basename(filename),str(e)))
    def _given_config_rpm_maker(self, keep_work_dir = False):
        self._cleanup_temp_dir()
        self.create_svn_repo()
        svn_service = SvnService(base_url=self.repo_url, username=None, password=None, path_to_config=config.get('svn_path_to_config'))

        if keep_work_dir:
            os.environ['KEEPWORKDIR'] = '1'
        elif os.environ.has_key('KEEPWORKDIR'):
            del os.environ['KEEPWORKDIR']

        return ConfigRpmMaker('2', svn_service)
 def _cleanup_temp_dir(self):
     temp_dir = config_dev.get('temp_dir')
     if temp_dir:
         if os.path.exists(temp_dir):
             shutil.rmtree(temp_dir)
         os.makedirs(temp_dir)
Example #20
0
 def test_get_hosts(self):
     service = SvnService(self.repo_url, None, None, path_to_config = config.get('svn_path_to_config'))
     self.assertEqual(['berweb01', 'devweb01', 'tuvweb01'], service.get_hosts(2))
    def build(self):
        self.logger.info("Building config rpm for host %s revision %s", self.hostname, self.revision)

        if os.path.exists(self.host_config_dir):
            raise Exception("ERROR: '%s' exists already whereas I should be creating it now." % self.host_config_dir)

        try:
            os.mkdir(self.host_config_dir)
        except Exception as e:
                raise CouldNotCreateConfigDirException("Could not create host config directory '%s' : %s" % self.host_config_dir, e)

        overall_requires = []
        overall_provides = []
        overall_svn_paths = []
        overall_exported = {}

        for segment in OVERLAY_ORDER:
            svn_paths, exported_paths, requires, provides = self._overlay_segment(segment)
            overall_exported[segment] = exported_paths
            overall_svn_paths += svn_paths
            overall_requires += requires
            overall_provides += provides

        self.logger.debug("Overall_exported: %s", str(overall_exported))
        self.logger.info("Overall_requires: %s", str(overall_requires))
        self.logger.info("Overall_provides: %s", str(overall_provides))
        self.logger.debug("Overall_svn_paths: %s", str(overall_svn_paths))

        if not os.path.exists(self.variables_dir):
            os.mkdir(self.variables_dir)

        self._write_dependency_file(overall_requires, self.rpm_requires_path, collapse_duplicates=True)
        self._write_dependency_file(overall_provides, self.rpm_provides_path, False)
        self._write_file(os.path.join(self.variables_dir, 'REVISION'), self.revision)

        repo_packages_regex = config.get('repo_packages_regex')
        if repo_packages_regex:
            self._write_dependency_file(overall_requires, os.path.join(self.variables_dir, 'RPM_REQUIRES_REPOS'), filter_regex=repo_packages_regex)
            self._write_dependency_file(overall_requires, os.path.join(self.variables_dir, 'RPM_REQUIRES_NON_REPOS'), filter_regex=repo_packages_regex, positive_filter=False)

        self._export_spec_file()
        self._save_log_entries_to_variable(overall_svn_paths)
        self._save_overlaying_to_variable(overall_exported)

        self._move_variables_out_of_rpm_dir()
        self._save_file_list()

        self._save_segment_variables()
        self._save_network_variables()

        patch_info = self._generate_patch_info()

        self._copy_files_for_config_viewer()

        # write patch info into variable and config viewer
        self._write_file(os.path.join(self.variables_dir, 'VARIABLES'), patch_info)
        self._write_file(os.path.join(self.config_viewer_host_dir, self.hostname + '.variables'), patch_info)

        self._filter_tokens_in_rpm_sources()

        self._build_rpm()

        self._filter_tokens_in_config_viewer()
        self._write_revision_file_for_config_viewer()
        self._write_overlaying_for_config_viewer(overall_exported)

        self._remove_logger_handlers()

        return self._find_rpms()
Example #22
0
 def test_log(self):
     service = SvnService(self.repo_url, None, None, path_to_config = config.get('svn_path_to_config'))
     logs = service.log('', 2, 5)
     self.assertEqual(1, len(logs))
Example #23
0
 def test_export(self):
     service = SvnService(self.repo_url, None, None, path_to_config = config.get('svn_path_to_config'))
     self.assertEqual([('host/berweb01', 'VARIABLES'), ('host/berweb01', 'VARIABLES/RPM_REQUIRES')], service.export('host/berweb01', 'target/tmp/test', 2))
 def create_svn_service_queue(self):
     svn_service = SvnService(base_url=self.repo_url, username=None, password=None,
                              path_to_config=config.get(KEY_SVN_PATH_TO_CONFIG))
     svn_service_queue = Queue()
     svn_service_queue.put(svn_service)
     return svn_service_queue
Example #25
0
 def test_get_change_set(self):
     service = SvnService(self.repo_url, None, None, path_to_config = config.get('svn_path_to_config'))
     self.assertEqual(['typ/web/data/index.html'], service.get_change_set(2))
Example #26
0
def log_execution_time_summaries(logging_function):
    logging_function('Execution times summary (keep in mind thread_count was set to %s):', get(KEY_THREAD_COUNT))

    for function_name in sorted(_summary.keys()):
        summary_of_function = _summary[function_name]
        rounded_elapsed_time = round_to_two_decimals_after_dot(summary_of_function[0])
        average_time = round_to_two_decimals_after_dot(summary_of_function[0] / summary_of_function[1])

        logging_function('    %3s times with average %5ss = sum %5ss : %s',
                         summary_of_function[1], average_time, rounded_elapsed_time, function_name)
from config_rpm_maker.configRpmMaker import ConfigRpmMaker
from config_rpm_maker.configRpmUploader import ConfigRpmUploader
from config_rpm_maker.configViewerAdapter import ConfigViewerAdapter
from config_rpm_maker.svn import SvnService
from config_rpm_maker.exceptions import BaseConfigRpmMakerException
from config_rpm_maker import config

import logging
import traceback
import sys


logging.basicConfig(format="%(asctime)s %(levelname)5s [%(name)s] - %(message)s", level=config.get("log_level", "INFO"))


class CliException(BaseConfigRpmMakerException):
    error_info = "Command Line Error:\n"


def mainMethod(args=sys.argv[1:]):
    try:
        if len(args) != 2:
            raise CliException("You need to provide 2 parameters (repo dir, revision).\nArguments were %s " % str(args))

        if not (args[1].isdigit() and int(args[1]) >= 0):
            raise CliException("Revision must be a positive integer.\nGiven revision was '%s'" % args[1])

        # first use case is post-commit hook. repo dir can be used as file:/// SVN URL
        svn_service = SvnService(base_url="file://" + args[0], path_to_config=config.get("svn_path_to_config"))
        config_rpm_maker = ConfigRpmMaker(revision=args[1], svn_service=svn_service)
        result = config_rpm_maker.build()
from config_rpm_maker.configRpmMaker import ConfigRpmMaker
from config_rpm_maker.svn import SvnService
from config_rpm_maker.exceptions import BaseConfigRpmMakerException
from config_rpm_maker import config

import logging
import traceback
import sys


logging.basicConfig(
    format="%(asctime)s %(levelname)5s [%(name)s] - %(message)s",
    level=config.get('log_level', 'INFO'),
)




class CliException(BaseConfigRpmMakerException):
    error_info = "Command Line Error:\n"

def mainMethod(args=sys.argv[1:]):
    try:
        if len(args) != 2:
            raise CliException("You need to provide 2 parameters (repo dir, revision).\nArguments were %s " % str(args))
    
        if not (args[1].isdigit() and int(args[1]) >= 0):
            raise CliException("Revision must be a positive integer.\nGiven revision was '%s'" % args[1])
        
    # first use case is post-commit hook. repo dir can be used as file:/// SVN URL
        svn_service = SvnService(