def _generate_xmljobs(self, options, jjb_config=None):
builder = Builder(jjb_config)
logger.info("Updating jobs in {0} ({1})".format(
options.path, options.names))
orig = time.time()
# Generate XML
parser = YamlParser(jjb_config)
registry = ModuleRegistry(jjb_config, builder.plugins_list)
xml_generator = XmlJobGenerator(registry)
parser.load_files(options.path)
registry.set_parser_data(parser.data)
job_data_list = parser.expandYaml(registry, options.names)
xml_jobs = xml_generator.generateXML(job_data_list)
jobs = parser.jobs
step = time.time()
logging.debug('%d XML files generated in %ss',
len(jobs), str(step - orig))
return builder, xml_jobs
def execute(options, config):
logger.debug("Config: {0}".format(config))
# check the ignore_cache setting: first from command line,
# if not present check from ini file
ignore_cache = False
if options.ignore_cache:
ignore_cache = options.ignore_cache
elif config.has_option('jenkins', 'ignore_cache'):
logging.warn('ignore_cache option should be moved to the [job_builder]'
' section in the config file, the one specified in the '
'[jenkins] section will be ignored in the future')
ignore_cache = config.getboolean('jenkins', 'ignore_cache')
elif config.has_option('job_builder', 'ignore_cache'):
ignore_cache = config.getboolean('job_builder', 'ignore_cache')
# workaround for python 2.6 interpolation error
# https://bugs.launchpad.net/openstack-ci/+bug/1259631
try:
user = config.get('jenkins', 'user')
except (TypeError, ConfigParser.NoOptionError):
user = None
try:
password = config.get('jenkins', 'password')
except (TypeError, ConfigParser.NoOptionError):
password = None
builder = Builder(config.get('jenkins', 'url'),
user,
password,
config,
ignore_cache=ignore_cache,
flush_cache=options.flush_cache)
if hasattr(options, 'path') and options.path == sys.stdin:
logger.debug("Input file is stdin")
if options.path.isatty():
key = 'CTRL+Z' if platform.system() == 'Windows' else 'CTRL+D'
logger.warn(
"Reading configuration from STDIN. Press %s to end input.",
key)
if options.command == 'delete':
for job in options.name:
logger.info("Deleting jobs in [{0}]".format(job))
builder.delete_job(job, options.path)
elif options.command == 'delete-all':
confirm('Sure you want to delete *ALL* jobs from Jenkins server?\n'
'(including those not managed by Jenkins Job Builder)')
logger.info("Deleting all jobs")
builder.delete_all_jobs()
elif options.command == 'update':
logger.info("Updating jobs in {0} ({1})".format(
options.path, options.names))
jobs = builder.update_job(options.path, options.names)
if options.delete_old:
builder.delete_old_managed(keep=[x.name for x in jobs])
elif options.command == 'test':
builder.update_job(options.path, options.name,
output=options.output_dir)
def assert_case(case_name):
case_source, case_result = (os.path.join(BASE_PATH, case_name + ext) for ext in ['.yml', '.xml'])
xml_output = StringIO()
builder = Builder('http://localhost:8080/', '', '', ignore_cache=True)
builder.update_jobs(case_source, output=xml_output, n_workers=1)
result_xml = ET.XML(xml_output.getvalue())
expected_xml = ET.XML(open(case_result).read())
assert ET.tostring(result_xml) == ET.tostring(expected_xml)
def get_devtools_jobs(devtools_fp):
builder = Builder("None", None, None, None, plugins_list={})
builder.load_files(devtools_fp)
builder.parser.expandYaml()
builder.parser.generateXML()
return [job.name for job in builder.parser.xml_jobs]
def execute(self, options, jjb_config):
builder = Builder(jjb_config)
if not utils.confirm(
'Sure you want to delete *ALL* jobs from Jenkins '
'server?\n(including those not managed by Jenkins '
'Job Builder)'):
sys.exit('Aborted')
logger.info("Deleting all jobs")
builder.delete_all_jobs()
def execute(self, options, jjb_config):
builder = Builder(jjb_config)
fn = options.path
registry = ModuleRegistry(jjb_config, builder.plugins_list)
for jobs_glob in options.name:
parser = YamlParser(jjb_config)
if fn:
parser.load_files(fn)
parser.expandYaml(registry, [jobs_glob])
jobs = [j['name'] for j in parser.jobs]
else:
jobs = [jobs_glob]
builder.delete_jobs(jobs)
def get_jjb_jobs(index_raw):
"""
Returns an array with job names. +index_raw+ can be a URL or a path.
This method uses the actual jenkins_jobs builder package to process the jobs.
"""
if index_raw.startswith('http'):
index_fp = urllib2.urlopen(index_raw)
else:
index_fp = open(index_raw, 'r')
builder = Builder("None", None, None, None, plugins_list={})
builder.load_files(index_fp)
builder.parser.expandYaml()
builder.parser.generateXML()
index_fp.close()
return [job.name for job in builder.parser.xml_jobs]
def assert_case(case_name):
case_source, case_result = (os.path.join(BASE_PATH, case_name + ext)
for ext in ['.yml', '.xml'])
jjb_config = JJBConfig()
builder = Builder(jjb_config)
# Generate XML
parser = YamlParser(jjb_config)
registry = ModuleRegistry(jjb_config, builder.plugins_list)
xml_generator = XmlJobGenerator(registry)
parser.load_files(case_source)
registry.set_parser_data(parser.data)
job_data_list = parser.expandYaml(registry, [])
xml_jobs = xml_generator.generateXML(job_data_list)
result_xml = ET.XML(xml_jobs[0].output())
expected_xml = ET.XML(open(case_result).read())
assert ET.tostring(result_xml) == ET.tostring(expected_xml)
def execute(options, config):
logger.debug("Config: {0}".format(config))
# check the ignore_cache setting: first from command line,
# if not present check from ini file
ignore_cache = False
if options.ignore_cache:
ignore_cache = options.ignore_cache
elif config.has_option('jenkins', 'ignore_cache'):
logging.warn('ignore_cache option should be moved to the [job_builder]'
' section in the config file, the one specified in the '
'[jenkins] section will be ignored in the future')
ignore_cache = config.getboolean('jenkins', 'ignore_cache')
elif config.has_option('job_builder', 'ignore_cache'):
ignore_cache = config.getboolean('job_builder', 'ignore_cache')
# workaround for python 2.6 interpolation error
# https://bugs.launchpad.net/openstack-ci/+bug/1259631
try:
user = config.get('jenkins', 'user')
except (TypeError, configparser.NoOptionError):
user = None
try:
password = config.get('jenkins', 'password')
except (TypeError, configparser.NoOptionError):
password = None
plugins_info = None
if getattr(options, 'plugins_info_path', None) is not None:
with open(options.plugins_info_path, 'r') as yaml_file:
plugins_info = yaml.load(yaml_file)
if not isinstance(plugins_info, list):
raise JenkinsJobsException("{0} must contain a Yaml list!".format(
options.plugins_info_path))
elif (not options.conf
or not config.getboolean("jenkins", "query_plugins_info")):
logger.debug("Skipping plugin info retrieval")
plugins_info = {}
if options.allow_empty_variables is not None:
config.set('job_builder', 'allow_empty_variables',
str(options.allow_empty_variables))
builder = Builder(config.get('jenkins', 'url'),
user,
password,
config,
ignore_cache=ignore_cache,
flush_cache=options.flush_cache,
plugins_list=plugins_info)
if getattr(options, 'path', None):
if options.path == sys.stdin:
logger.debug("Input file is stdin")
if options.path.isatty():
key = 'CTRL+Z' if platform.system() == 'Windows' else 'CTRL+D'
logger.warn(
"Reading configuration from STDIN. Press %s to end input.",
key)
# take list of paths
options.path = options.path.split(os.pathsep)
do_recurse = (getattr(options, 'recursive', False)
or config.getboolean('job_builder', 'recursive'))
excludes = [e for elist in options.exclude
for e in elist.split(os.pathsep)] or \
config.get('job_builder', 'exclude').split(os.pathsep)
paths = []
for path in options.path:
if do_recurse and os.path.isdir(path):
paths.extend(recurse_path(path, excludes))
else:
paths.append(path)
options.path = paths
if options.command == 'delete':
if options.del_views:
for view in options.name:
builder.delete_view(view, options.path)
else:
for job in options.name:
builder.delete_job(job, options.path)
elif options.command == 'delete-all':
deleting_jobs = False
deleting_views = False
if options.only_jobs == options.only_views:
# if (delete-all --jobs --views) or (delete-all)
# Equivalent of {NOT(--jobs XOR --views)} only passes if both
# subparameters are true or both false
reach = 'jobs AND views'
deleting_jobs = True
deleting_views = True
elif options.only_jobs and not options.only_views:
reach = 'jobs'
deleting_jobs = True
elif options.only_views and not options.only_jobs:
reach = 'views'
deleting_views = True
confirm(
'Sure you want to delete *ALL* %s from Jenkins server?\n'
'(including those not managed by Jenkins Job Builder)', reach)
if deleting_jobs:
logger.info("Deleting all jobs")
builder.delete_all_jobs()
if deleting_views:
logger.info("Deleting all views")
builder.delete_all_views()
elif options.command == 'update':
logger.info("Updating jobs in {0} ({1})".format(
options.path, options.names))
jobs, views, num_updated_jobs, num_updated_views = builder.update_job(
options.path, options.names)
logger.info("%d jobs updated, %d views updated", num_updated_jobs,
num_updated_views)
if options.delete_old:
num_deleted_jobs = builder.delete_old_managed()
logger.info("Number of jobs deleted: %d", num_deleted_jobs)
elif options.command == 'test':
builder.update_job(options.path,
options.name,
output=options.output_dir)
def execute(options, config):
logger.debug("Config: {0}".format(config))
# check the ignore_cache setting: first from command line,
# if not present check from ini file
ignore_cache = False
if options.ignore_cache:
ignore_cache = options.ignore_cache
elif config.has_option('jenkins', 'ignore_cache'):
logging.warn('ignore_cache option should be moved to the [job_builder]'
' section in the config file, the one specified in the '
'[jenkins] section will be ignored in the future')
ignore_cache = config.getboolean('jenkins', 'ignore_cache')
elif config.has_option('job_builder', 'ignore_cache'):
ignore_cache = config.getboolean('job_builder', 'ignore_cache')
# Jenkins supports access as an anonymous user, which can be used to
# ensure read-only behaviour when querying the version of plugins
# installed for test mode to generate XML output matching what will be
# uploaded. To enable must pass 'None' as the value for user and password
# to python-jenkins
#
# catching 'TypeError' is a workaround for python 2.6 interpolation error
# https://bugs.launchpad.net/openstack-ci/+bug/1259631
try:
user = config.get('jenkins', 'user')
except (TypeError, configparser.NoOptionError):
user = None
try:
password = config.get('jenkins', 'password')
except (TypeError, configparser.NoOptionError):
password = None
# Inform the user as to what is likely to happen, as they may specify
# a real jenkins instance in test mode to get the plugin info to check
# the XML generated.
if user is None and password is None:
logger.info("Will use anonymous access to Jenkins if needed.")
elif (user is not None and password is None) or (
user is None and password is not None):
raise JenkinsJobsException(
"Cannot authenticate to Jenkins with only one of User and "
"Password provided, please check your configuration."
)
# None -- no timeout, blocking mode; same as setblocking(True)
# 0.0 -- non-blocking mode; same as setblocking(False) <--- default
# > 0 -- timeout mode; operations time out after timeout seconds
# < 0 -- illegal; raises an exception
# to retain the default must use
# "timeout=jenkins_jobs.builder._DEFAULT_TIMEOUT" or not set timeout at
# all.
timeout = jenkins_jobs.builder._DEFAULT_TIMEOUT
try:
timeout = config.getfloat('jenkins', 'timeout')
except (ValueError):
raise JenkinsJobsException("Jenkins timeout config is invalid")
except (TypeError, configparser.NoOptionError):
pass
plugins_info = None
if getattr(options, 'plugins_info_path', None) is not None:
with io.open(options.plugins_info_path, 'r',
encoding='utf-8') as yaml_file:
plugins_info = yaml.load(yaml_file)
if not isinstance(plugins_info, list):
raise JenkinsJobsException("{0} must contain a Yaml list!"
.format(options.plugins_info_path))
elif (not options.conf or not
config.getboolean("jenkins", "query_plugins_info")):
logger.debug("Skipping plugin info retrieval")
plugins_info = {}
if options.allow_empty_variables is not None:
config.set('job_builder',
'allow_empty_variables',
str(options.allow_empty_variables))
builder = Builder(config.get('jenkins', 'url'),
user,
password,
config,
jenkins_timeout=timeout,
ignore_cache=ignore_cache,
flush_cache=options.flush_cache,
plugins_list=plugins_info)
if getattr(options, 'path', None):
if hasattr(options.path, 'read'):
logger.debug("Input file is stdin")
if options.path.isatty():
key = 'CTRL+Z' if platform.system() == 'Windows' else 'CTRL+D'
logger.warn(
"Reading configuration from STDIN. Press %s to end input.",
key)
else:
# take list of paths
options.path = options.path.split(os.pathsep)
do_recurse = (getattr(options, 'recursive', False) or
config.getboolean('job_builder', 'recursive'))
excludes = [e for elist in options.exclude
for e in elist.split(os.pathsep)] or \
config.get('job_builder', 'exclude').split(os.pathsep)
paths = []
for path in options.path:
if do_recurse and os.path.isdir(path):
paths.extend(recurse_path(path, excludes))
else:
paths.append(path)
options.path = paths
if options.command == 'delete':
for job in options.name:
builder.delete_job(job, options.path)
elif options.command == 'delete-all':
confirm('Sure you want to delete *ALL* jobs from Jenkins server?\n'
'(including those not managed by Jenkins Job Builder)')
logger.info("Deleting all jobs")
builder.delete_all_jobs()
elif options.command == 'update':
if options.n_workers < 0:
raise JenkinsJobsException(
'Number of workers must be equal or greater than 0')
logger.info("Updating jobs in {0} ({1})".format(
options.path, options.names))
jobs, num_updated_jobs = builder.update_jobs(
options.path, options.names,
n_workers=options.n_workers)
logger.info("Number of jobs updated: %d", num_updated_jobs)
if options.delete_old:
num_deleted_jobs = builder.delete_old_managed()
logger.info("Number of jobs deleted: %d", num_deleted_jobs)
elif options.command == 'test':
builder.update_jobs(options.path, options.name,
output=options.output_dir,
n_workers=1)
def execute(options, config):
logger.debug("Config: {0}".format(config))
# check the ignore_cache setting: first from command line,
# if not present check from ini file
ignore_cache = False
if options.ignore_cache:
ignore_cache = options.ignore_cache
elif config.has_option('jenkins', 'ignore_cache'):
logging.warn('ignore_cache option should be moved to the [job_builder]'
' section in the config file, the one specified in the '
'[jenkins] section will be ignored in the future')
ignore_cache = config.getboolean('jenkins', 'ignore_cache')
elif config.has_option('job_builder', 'ignore_cache'):
ignore_cache = config.getboolean('job_builder', 'ignore_cache')
# workaround for python 2.6 interpolation error
# https://bugs.launchpad.net/openstack-ci/+bug/1259631
try:
user = config.get('jenkins', 'user')
except (TypeError, configparser.NoOptionError):
user = None
try:
password = config.get('jenkins', 'password')
except (TypeError, configparser.NoOptionError):
password = None
plugins_info = None
if getattr(options, 'plugins_info_path', None) is not None:
with open(options.plugins_info_path, 'r') as yaml_file:
plugins_info = yaml.load(yaml_file)
if not isinstance(plugins_info, list):
raise JenkinsJobsException("{0} must contain a Yaml list!".format(
options.plugins_info_path))
builder = Builder(config.get('jenkins', 'url'),
user,
password,
config,
ignore_cache=ignore_cache,
flush_cache=options.flush_cache,
plugins_list=plugins_info)
if getattr(options, 'path', None):
if options.path == sys.stdin:
logger.debug("Input file is stdin")
if options.path.isatty():
key = 'CTRL+Z' if platform.system() == 'Windows' else 'CTRL+D'
logger.warn(
"Reading configuration from STDIN. Press %s to end input.",
key)
# take list of paths
options.path = options.path.split(os.pathsep)
do_recurse = (getattr(options, 'recursive', False)
or config.getboolean('job_builder', 'recursive'))
paths = []
for path in options.path:
if do_recurse and os.path.isdir(path):
paths.extend(recurse_path(path))
else:
paths.append(path)
options.path = paths
if options.command == 'delete':
for job in options.name:
builder.delete_job(job, options.path)
elif options.command == 'delete-all':
confirm('Sure you want to delete *ALL* jobs from Jenkins server?\n'
'(including those not managed by Jenkins Job Builder)')
logger.info("Deleting all jobs")
builder.delete_all_jobs()
elif options.command == 'update':
logger.info("Updating jobs in {0} ({1})".format(
options.path, options.names))
jobs = builder.update_job(options.path, options.names)
if options.delete_old:
builder.delete_old_managed(keep=[x.name for x in jobs])
elif options.command == 'test':
builder.update_job(options.path,
options.name,
output=options.output_dir)
jenkins_password = "******"
# create a ConfigParser object by hand
config = configparser.ConfigParser()
config.add_section("job_builder")
config.set("job_builder", "keep_descriptions", "False")
config.set("job_builder", "ignore_cache", "False")
config.set("job_builder", "recursive", "False")
config.set("job_builder", "exclude", ".*")
config.set("job_builder", "allow_duplicates", "False")
config.set("job_builder", "allow_empty_variables", "False")
config.add_section("jenkins")
config.set("jenkins", "url", jenkins_url)
config.set("jenkins", "user", jenkins_user)
config.set("jenkins", "password", jenkins_password)
config.set("jenkins", "query_plugins_info", "False")
user = config.get('jenkins', 'user')
password = config.get('jenkins', 'password')
builder = Builder(jenkins_url,
user,
password,
config,
jenkins_timeout=10,
ignore_cache=True,
flush_cache=True,
plugins_list={})
jobs, num_updated_jobs = builder.update_jobs([sys.argv[1]], [], n_workers=2)