Пример #1
0
    def _generate_xmljobs(self, options, jjb_config=None):
        builder = Builder(jjb_config)

        logger.info("Updating jobs in {0} ({1})".format(
            options.path, options.names))
        orig = time.time()

        # Generate XML
        parser = YamlParser(jjb_config)
        registry = ModuleRegistry(jjb_config, builder.plugins_list)
        xml_generator = XmlJobGenerator(registry)

        parser.load_files(options.path)
        registry.set_parser_data(parser.data)

        job_data_list = parser.expandYaml(registry, options.names)

        xml_jobs = xml_generator.generateXML(job_data_list)

        jobs = parser.jobs
        step = time.time()
        logging.debug('%d XML files generated in %ss',
                      len(jobs), str(step - orig))

        return builder, xml_jobs
Пример #2
0
def execute(options, config):
    logger.debug("Config: {0}".format(config))

    # check the ignore_cache setting: first from command line,
    # if not present check from ini file
    ignore_cache = False
    if options.ignore_cache:
        ignore_cache = options.ignore_cache
    elif config.has_option('jenkins', 'ignore_cache'):
        logging.warn('ignore_cache option should be moved to the [job_builder]'
                     ' section in the config file, the one specified in the '
                     '[jenkins] section will be ignored in the future')
        ignore_cache = config.getboolean('jenkins', 'ignore_cache')
    elif config.has_option('job_builder', 'ignore_cache'):
        ignore_cache = config.getboolean('job_builder', 'ignore_cache')

    # workaround for python 2.6 interpolation error
    # https://bugs.launchpad.net/openstack-ci/+bug/1259631
    try:
        user = config.get('jenkins', 'user')
    except (TypeError, ConfigParser.NoOptionError):
        user = None
    try:
        password = config.get('jenkins', 'password')
    except (TypeError, ConfigParser.NoOptionError):
        password = None

    builder = Builder(config.get('jenkins', 'url'),
                      user,
                      password,
                      config,
                      ignore_cache=ignore_cache,
                      flush_cache=options.flush_cache)

    if hasattr(options, 'path') and options.path == sys.stdin:
        logger.debug("Input file is stdin")
        if options.path.isatty():
            key = 'CTRL+Z' if platform.system() == 'Windows' else 'CTRL+D'
            logger.warn(
                "Reading configuration from STDIN. Press %s to end input.",
                key)

    if options.command == 'delete':
        for job in options.name:
            logger.info("Deleting jobs in [{0}]".format(job))
            builder.delete_job(job, options.path)
    elif options.command == 'delete-all':
        confirm('Sure you want to delete *ALL* jobs from Jenkins server?\n'
                '(including those not managed by Jenkins Job Builder)')
        logger.info("Deleting all jobs")
        builder.delete_all_jobs()
    elif options.command == 'update':
        logger.info("Updating jobs in {0} ({1})".format(
            options.path, options.names))
        jobs = builder.update_job(options.path, options.names)
        if options.delete_old:
            builder.delete_old_managed(keep=[x.name for x in jobs])
    elif options.command == 'test':
        builder.update_job(options.path, options.name,
                           output=options.output_dir)
Пример #3
0
def assert_case(case_name):
    case_source, case_result = (os.path.join(BASE_PATH, case_name + ext) for ext in ['.yml', '.xml'])
    xml_output = StringIO()
    builder = Builder('http://localhost:8080/', '', '', ignore_cache=True)
    builder.update_jobs(case_source, output=xml_output, n_workers=1)
    result_xml = ET.XML(xml_output.getvalue())
    expected_xml = ET.XML(open(case_result).read())
    assert ET.tostring(result_xml) == ET.tostring(expected_xml)
Пример #4
0
def get_devtools_jobs(devtools_fp):
    builder = Builder("None", None, None, None, plugins_list={})

    builder.load_files(devtools_fp)
    builder.parser.expandYaml()
    builder.parser.generateXML()

    return [job.name for job in builder.parser.xml_jobs]
Пример #5
0
    def execute(self, options, jjb_config):
        builder = Builder(jjb_config)

        if not utils.confirm(
                'Sure you want to delete *ALL* jobs from Jenkins '
                'server?\n(including those not managed by Jenkins '
                'Job Builder)'):
            sys.exit('Aborted')

        logger.info("Deleting all jobs")
        builder.delete_all_jobs()
Пример #6
0
    def execute(self, options, jjb_config):
        builder = Builder(jjb_config)

        fn = options.path

        registry = ModuleRegistry(jjb_config, builder.plugins_list)
        for jobs_glob in options.name:
            parser = YamlParser(jjb_config)

            if fn:
                parser.load_files(fn)
                parser.expandYaml(registry, [jobs_glob])
                jobs = [j['name'] for j in parser.jobs]
            else:
                jobs = [jobs_glob]

            builder.delete_jobs(jobs)
Пример #7
0
def get_jjb_jobs(index_raw):
    """
    Returns an array with job names. +index_raw+ can be a URL or a path.
    This method uses the actual jenkins_jobs builder package to process the jobs.
    """

    if index_raw.startswith('http'):
        index_fp = urllib2.urlopen(index_raw)
    else:
        index_fp = open(index_raw, 'r')

    builder = Builder("None", None, None, None, plugins_list={})

    builder.load_files(index_fp)
    builder.parser.expandYaml()
    builder.parser.generateXML()

    index_fp.close()

    return [job.name for job in builder.parser.xml_jobs]
Пример #8
0
def assert_case(case_name):
    case_source, case_result = (os.path.join(BASE_PATH, case_name + ext)
                                for ext in ['.yml', '.xml'])
    jjb_config = JJBConfig()
    builder = Builder(jjb_config)

    # Generate XML
    parser = YamlParser(jjb_config)
    registry = ModuleRegistry(jjb_config, builder.plugins_list)
    xml_generator = XmlJobGenerator(registry)

    parser.load_files(case_source)
    registry.set_parser_data(parser.data)

    job_data_list = parser.expandYaml(registry, [])

    xml_jobs = xml_generator.generateXML(job_data_list)

    result_xml = ET.XML(xml_jobs[0].output())
    expected_xml = ET.XML(open(case_result).read())
    assert ET.tostring(result_xml) == ET.tostring(expected_xml)
Пример #9
0
def execute(options, config):
    logger.debug("Config: {0}".format(config))

    # check the ignore_cache setting: first from command line,
    # if not present check from ini file
    ignore_cache = False
    if options.ignore_cache:
        ignore_cache = options.ignore_cache
    elif config.has_option('jenkins', 'ignore_cache'):
        logging.warn('ignore_cache option should be moved to the [job_builder]'
                     ' section in the config file, the one specified in the '
                     '[jenkins] section will be ignored in the future')
        ignore_cache = config.getboolean('jenkins', 'ignore_cache')
    elif config.has_option('job_builder', 'ignore_cache'):
        ignore_cache = config.getboolean('job_builder', 'ignore_cache')

    # workaround for python 2.6 interpolation error
    # https://bugs.launchpad.net/openstack-ci/+bug/1259631
    try:
        user = config.get('jenkins', 'user')
    except (TypeError, configparser.NoOptionError):
        user = None
    try:
        password = config.get('jenkins', 'password')
    except (TypeError, configparser.NoOptionError):
        password = None

    plugins_info = None

    if getattr(options, 'plugins_info_path', None) is not None:
        with open(options.plugins_info_path, 'r') as yaml_file:
            plugins_info = yaml.load(yaml_file)
        if not isinstance(plugins_info, list):
            raise JenkinsJobsException("{0} must contain a Yaml list!".format(
                options.plugins_info_path))
    elif (not options.conf
          or not config.getboolean("jenkins", "query_plugins_info")):
        logger.debug("Skipping plugin info retrieval")
        plugins_info = {}

    if options.allow_empty_variables is not None:
        config.set('job_builder', 'allow_empty_variables',
                   str(options.allow_empty_variables))

    builder = Builder(config.get('jenkins', 'url'),
                      user,
                      password,
                      config,
                      ignore_cache=ignore_cache,
                      flush_cache=options.flush_cache,
                      plugins_list=plugins_info)

    if getattr(options, 'path', None):
        if options.path == sys.stdin:
            logger.debug("Input file is stdin")
            if options.path.isatty():
                key = 'CTRL+Z' if platform.system() == 'Windows' else 'CTRL+D'
                logger.warn(
                    "Reading configuration from STDIN. Press %s to end input.",
                    key)

        # take list of paths
        options.path = options.path.split(os.pathsep)

        do_recurse = (getattr(options, 'recursive', False)
                      or config.getboolean('job_builder', 'recursive'))

        excludes = [e for elist in options.exclude
                    for e in elist.split(os.pathsep)] or \
            config.get('job_builder', 'exclude').split(os.pathsep)
        paths = []
        for path in options.path:
            if do_recurse and os.path.isdir(path):
                paths.extend(recurse_path(path, excludes))
            else:
                paths.append(path)
        options.path = paths

    if options.command == 'delete':
        if options.del_views:
            for view in options.name:
                builder.delete_view(view, options.path)
        else:
            for job in options.name:
                builder.delete_job(job, options.path)

    elif options.command == 'delete-all':
        deleting_jobs = False
        deleting_views = False
        if options.only_jobs == options.only_views:
            # if (delete-all --jobs --views) or (delete-all)
            # Equivalent of {NOT(--jobs XOR --views)} only passes if both
            # subparameters are true or both false
            reach = 'jobs AND views'
            deleting_jobs = True
            deleting_views = True
        elif options.only_jobs and not options.only_views:
            reach = 'jobs'
            deleting_jobs = True
        elif options.only_views and not options.only_jobs:
            reach = 'views'
            deleting_views = True
        confirm(
            'Sure you want to delete *ALL* %s from Jenkins server?\n'
            '(including those not managed by Jenkins Job Builder)', reach)
        if deleting_jobs:
            logger.info("Deleting all jobs")
            builder.delete_all_jobs()
        if deleting_views:
            logger.info("Deleting all views")
            builder.delete_all_views()

    elif options.command == 'update':
        logger.info("Updating jobs in {0} ({1})".format(
            options.path, options.names))
        jobs, views, num_updated_jobs, num_updated_views = builder.update_job(
            options.path, options.names)
        logger.info("%d jobs updated, %d views updated", num_updated_jobs,
                    num_updated_views)
        if options.delete_old:
            num_deleted_jobs = builder.delete_old_managed()
            logger.info("Number of jobs deleted: %d", num_deleted_jobs)
    elif options.command == 'test':
        builder.update_job(options.path,
                           options.name,
                           output=options.output_dir)
Пример #10
0
def execute(options, config):
    logger.debug("Config: {0}".format(config))

    # check the ignore_cache setting: first from command line,
    # if not present check from ini file
    ignore_cache = False
    if options.ignore_cache:
        ignore_cache = options.ignore_cache
    elif config.has_option('jenkins', 'ignore_cache'):
        logging.warn('ignore_cache option should be moved to the [job_builder]'
                     ' section in the config file, the one specified in the '
                     '[jenkins] section will be ignored in the future')
        ignore_cache = config.getboolean('jenkins', 'ignore_cache')
    elif config.has_option('job_builder', 'ignore_cache'):
        ignore_cache = config.getboolean('job_builder', 'ignore_cache')

    # Jenkins supports access as an anonymous user, which can be used to
    # ensure read-only behaviour when querying the version of plugins
    # installed for test mode to generate XML output matching what will be
    # uploaded. To enable must pass 'None' as the value for user and password
    # to python-jenkins
    #
    # catching 'TypeError' is a workaround for python 2.6 interpolation error
    # https://bugs.launchpad.net/openstack-ci/+bug/1259631
    try:
        user = config.get('jenkins', 'user')
    except (TypeError, configparser.NoOptionError):
        user = None

    try:
        password = config.get('jenkins', 'password')
    except (TypeError, configparser.NoOptionError):
        password = None

    # Inform the user as to what is likely to happen, as they may specify
    # a real jenkins instance in test mode to get the plugin info to check
    # the XML generated.
    if user is None and password is None:
        logger.info("Will use anonymous access to Jenkins if needed.")
    elif (user is not None and password is None) or (
            user is None and password is not None):
        raise JenkinsJobsException(
            "Cannot authenticate to Jenkins with only one of User and "
            "Password provided, please check your configuration."
        )

    # None -- no timeout, blocking mode; same as setblocking(True)
    # 0.0 -- non-blocking mode; same as setblocking(False) <--- default
    # > 0 -- timeout mode; operations time out after timeout seconds
    # < 0 -- illegal; raises an exception
    # to retain the default must use
    # "timeout=jenkins_jobs.builder._DEFAULT_TIMEOUT" or not set timeout at
    # all.
    timeout = jenkins_jobs.builder._DEFAULT_TIMEOUT
    try:
        timeout = config.getfloat('jenkins', 'timeout')
    except (ValueError):
        raise JenkinsJobsException("Jenkins timeout config is invalid")
    except (TypeError, configparser.NoOptionError):
        pass

    plugins_info = None

    if getattr(options, 'plugins_info_path', None) is not None:
        with io.open(options.plugins_info_path, 'r',
                     encoding='utf-8') as yaml_file:
            plugins_info = yaml.load(yaml_file)
        if not isinstance(plugins_info, list):
            raise JenkinsJobsException("{0} must contain a Yaml list!"
                                       .format(options.plugins_info_path))
    elif (not options.conf or not
          config.getboolean("jenkins", "query_plugins_info")):
        logger.debug("Skipping plugin info retrieval")
        plugins_info = {}

    if options.allow_empty_variables is not None:
        config.set('job_builder',
                   'allow_empty_variables',
                   str(options.allow_empty_variables))

    builder = Builder(config.get('jenkins', 'url'),
                      user,
                      password,
                      config,
                      jenkins_timeout=timeout,
                      ignore_cache=ignore_cache,
                      flush_cache=options.flush_cache,
                      plugins_list=plugins_info)

    if getattr(options, 'path', None):
        if hasattr(options.path, 'read'):
            logger.debug("Input file is stdin")
            if options.path.isatty():
                key = 'CTRL+Z' if platform.system() == 'Windows' else 'CTRL+D'
                logger.warn(
                    "Reading configuration from STDIN. Press %s to end input.",
                    key)
        else:
            # take list of paths
            options.path = options.path.split(os.pathsep)

            do_recurse = (getattr(options, 'recursive', False) or
                          config.getboolean('job_builder', 'recursive'))

            excludes = [e for elist in options.exclude
                        for e in elist.split(os.pathsep)] or \
                config.get('job_builder', 'exclude').split(os.pathsep)
            paths = []
            for path in options.path:
                if do_recurse and os.path.isdir(path):
                    paths.extend(recurse_path(path, excludes))
                else:
                    paths.append(path)
            options.path = paths

    if options.command == 'delete':
        for job in options.name:
            builder.delete_job(job, options.path)
    elif options.command == 'delete-all':
        confirm('Sure you want to delete *ALL* jobs from Jenkins server?\n'
                '(including those not managed by Jenkins Job Builder)')
        logger.info("Deleting all jobs")
        builder.delete_all_jobs()
    elif options.command == 'update':
        if options.n_workers < 0:
            raise JenkinsJobsException(
                'Number of workers must be equal or greater than 0')

        logger.info("Updating jobs in {0} ({1})".format(
            options.path, options.names))
        jobs, num_updated_jobs = builder.update_jobs(
            options.path, options.names,
            n_workers=options.n_workers)
        logger.info("Number of jobs updated: %d", num_updated_jobs)
        if options.delete_old:
            num_deleted_jobs = builder.delete_old_managed()
            logger.info("Number of jobs deleted: %d", num_deleted_jobs)
    elif options.command == 'test':
        builder.update_jobs(options.path, options.name,
                            output=options.output_dir,
                            n_workers=1)
Пример #11
0
def execute(options, config):
    logger.debug("Config: {0}".format(config))

    # check the ignore_cache setting: first from command line,
    # if not present check from ini file
    ignore_cache = False
    if options.ignore_cache:
        ignore_cache = options.ignore_cache
    elif config.has_option('jenkins', 'ignore_cache'):
        logging.warn('ignore_cache option should be moved to the [job_builder]'
                     ' section in the config file, the one specified in the '
                     '[jenkins] section will be ignored in the future')
        ignore_cache = config.getboolean('jenkins', 'ignore_cache')
    elif config.has_option('job_builder', 'ignore_cache'):
        ignore_cache = config.getboolean('job_builder', 'ignore_cache')

    # workaround for python 2.6 interpolation error
    # https://bugs.launchpad.net/openstack-ci/+bug/1259631
    try:
        user = config.get('jenkins', 'user')
    except (TypeError, configparser.NoOptionError):
        user = None
    try:
        password = config.get('jenkins', 'password')
    except (TypeError, configparser.NoOptionError):
        password = None

    plugins_info = None

    if getattr(options, 'plugins_info_path', None) is not None:
        with open(options.plugins_info_path, 'r') as yaml_file:
            plugins_info = yaml.load(yaml_file)
        if not isinstance(plugins_info, list):
            raise JenkinsJobsException("{0} must contain a Yaml list!".format(
                options.plugins_info_path))

    builder = Builder(config.get('jenkins', 'url'),
                      user,
                      password,
                      config,
                      ignore_cache=ignore_cache,
                      flush_cache=options.flush_cache,
                      plugins_list=plugins_info)

    if getattr(options, 'path', None):
        if options.path == sys.stdin:
            logger.debug("Input file is stdin")
            if options.path.isatty():
                key = 'CTRL+Z' if platform.system() == 'Windows' else 'CTRL+D'
                logger.warn(
                    "Reading configuration from STDIN. Press %s to end input.",
                    key)

        # take list of paths
        options.path = options.path.split(os.pathsep)

        do_recurse = (getattr(options, 'recursive', False)
                      or config.getboolean('job_builder', 'recursive'))

        paths = []
        for path in options.path:
            if do_recurse and os.path.isdir(path):
                paths.extend(recurse_path(path))
            else:
                paths.append(path)
        options.path = paths

    if options.command == 'delete':
        for job in options.name:
            builder.delete_job(job, options.path)
    elif options.command == 'delete-all':
        confirm('Sure you want to delete *ALL* jobs from Jenkins server?\n'
                '(including those not managed by Jenkins Job Builder)')
        logger.info("Deleting all jobs")
        builder.delete_all_jobs()
    elif options.command == 'update':
        logger.info("Updating jobs in {0} ({1})".format(
            options.path, options.names))
        jobs = builder.update_job(options.path, options.names)
        if options.delete_old:
            builder.delete_old_managed(keep=[x.name for x in jobs])
    elif options.command == 'test':
        builder.update_job(options.path,
                           options.name,
                           output=options.output_dir)
Пример #12
0
jenkins_password = "******"

# create a ConfigParser object by hand
config = configparser.ConfigParser()
config.add_section("job_builder")
config.set("job_builder", "keep_descriptions", "False")
config.set("job_builder", "ignore_cache", "False")
config.set("job_builder", "recursive", "False")
config.set("job_builder", "exclude", ".*")
config.set("job_builder", "allow_duplicates", "False")
config.set("job_builder", "allow_empty_variables", "False")
config.add_section("jenkins")
config.set("jenkins", "url", jenkins_url)
config.set("jenkins", "user", jenkins_user)
config.set("jenkins", "password", jenkins_password)
config.set("jenkins", "query_plugins_info", "False")

user = config.get('jenkins', 'user')
password = config.get('jenkins', 'password')

builder = Builder(jenkins_url,
                  user,
                  password,
                  config,
                  jenkins_timeout=10,
                  ignore_cache=True,
                  flush_cache=True,
                  plugins_list={})

jobs, num_updated_jobs = builder.update_jobs([sys.argv[1]], [], n_workers=2)