def build_condition(cdata):
kind = cdata['condition-kind']
ctag = XML.SubElement(root_tag, condition_tag)
if kind == "always":
ctag.set('class',
'org.jenkins_ci.plugins.run_condition.core.AlwaysRun')
elif kind == "never":
ctag.set('class',
'org.jenkins_ci.plugins.run_condition.core.NeverRun')
elif kind == "boolean-expression":
ctag.set(
'class', 'org.jenkins_ci.plugins.run_condition.core.'
'BooleanCondition')
XML.SubElement(ctag, "token").text = cdata['condition-expression']
elif kind == "current-status":
ctag.set(
'class', 'org.jenkins_ci.plugins.run_condition.core.'
'StatusCondition')
wr = XML.SubElement(ctag, 'worstResult')
wr_name = cdata['condition-worst']
if wr_name not in hudson_model.THRESHOLDS:
raise JenkinsJobsException(
"threshold must be one of %s" %
", ".join(hudson_model.THRESHOLDS.keys()))
wr_threshold = hudson_model.THRESHOLDS[wr_name]
XML.SubElement(wr, "name").text = wr_threshold['name']
XML.SubElement(wr, "ordinal").text = wr_threshold['ordinal']
XML.SubElement(wr, "color").text = wr_threshold['color']
XML.SubElement(wr, "completeBuild").text = \
str(wr_threshold['complete']).lower()
br = XML.SubElement(ctag, 'bestResult')
br_name = cdata['condition-best']
if not br_name in hudson_model.THRESHOLDS:
raise JenkinsJobsException(
"threshold must be one of %s" %
", ".join(hudson_model.THRESHOLDS.keys()))
br_threshold = hudson_model.THRESHOLDS[br_name]
XML.SubElement(br, "name").text = br_threshold['name']
XML.SubElement(br, "ordinal").text = br_threshold['ordinal']
XML.SubElement(br, "color").text = br_threshold['color']
XML.SubElement(br, "completeBuild").text = \
str(wr_threshold['complete']).lower()
elif kind == "shell":
ctag.set(
'class', 'org.jenkins_ci.plugins.run_condition.contributed.'
'ShellCondition')
XML.SubElement(ctag, "command").text = cdata['condition-command']
elif kind == "windows-shell":
ctag.set(
'class', 'org.jenkins_ci.plugins.run_condition.contributed.'
'BatchFileCondition')
XML.SubElement(ctag, "command").text = cdata['condition-command']
elif kind == "file-exists":
ctag.set(
'class', 'org.jenkins_ci.plugins.run_condition.core.'
'FileExistsCondition')
XML.SubElement(ctag, "file").text = cdata['condition-filename']
basedir = cdata.get('condition-basedir', 'workspace')
basedir_tag = XML.SubElement(ctag, "baseDir")
if "workspace" == basedir:
basedir_tag.set(
'class', 'org.jenkins_ci.plugins.run_condition.common.'
'BaseDirectory$Workspace')
elif "artifact-directory" == basedir:
basedir_tag.set(
'class', 'org.jenkins_ci.plugins.run_condition.common.'
'BaseDirectory$ArtifactsDir')
elif "jenkins-home" == basedir:
basedir_tag.set(
'class', 'org.jenkins_ci.plugins.run_condition.common.'
'BaseDirectory$JenkinsHome')
def build_trends_publisher(plugin_name, xml_element, data):
"""Helper to create various trend publishers.
"""
def append_thresholds(element, data, only_totals):
"""Appends the status thresholds.
"""
for status in ['unstable', 'failed']:
status_data = data.get(status, {})
limits = [
('total-all', 'TotalAll'),
('total-high', 'TotalHigh'),
('total-normal', 'TotalNormal'),
('total-low', 'TotalLow')]
if only_totals is False:
limits.extend([
('new-all', 'NewAll'),
('new-high', 'NewHigh'),
('new-normal', 'NewNormal'),
('new-low', 'NewLow')])
for key, tag_suffix in limits:
tag_name = status + tag_suffix
XML.SubElement(element, tag_name).text = str(
status_data.get(key, ''))
# Tuples containing: setting name, tag name, default value
settings = [
('healthy', 'healthy', ''),
('unhealthy', 'unHealthy', ''),
('health-threshold', 'thresholdLimit', 'low'),
('plugin-name', 'pluginName', plugin_name),
('default-encoding', 'defaultEncoding', ''),
('can-run-on-failed', 'canRunOnFailed', False),
('use-stable-build-as-reference', 'useStableBuildAsReference', False),
('use-delta-values', 'useDeltaValues', False),
('thresholds', 'thresholds', {}),
('should-detect-modules', 'shouldDetectModules', False),
('dont-compute-new', 'dontComputeNew', True),
('do-not-resolve-relative-paths', 'doNotResolveRelativePaths', False),
('pattern', 'pattern', '')]
thresholds = ['low', 'normal', 'high']
for key, tag_name, default in settings:
xml_config = XML.SubElement(xml_element, tag_name)
config_value = data.get(key, default)
if key == 'thresholds':
append_thresholds(
xml_config,
config_value,
data.get('dont-compute-new', True))
elif key == 'health-threshold' and config_value not in thresholds:
raise JenkinsJobsException("health-threshold must be one of %s" %
", ".join(thresholds))
elif key == 'use-stable-build-as-reference' \
and '[DependencyCheck]' in plugin_name:
XML.SubElement(xml_element, 'usePreviousBuildAsReference') \
.text = \
str(data.get('use-previous-build-as-reference', False)) \
.lower()
xml_config.text = str(config_value).lower()
else:
if isinstance(default, bool):
xml_config.text = str(config_value).lower()
else:
xml_config.text = str(config_value)
def generateXML(self, jobs_filter=None):
changed = True
while changed:
changed = False
for module in self.registry.modules:
if hasattr(module, 'handle_data'):
if module.handle_data(self):
changed = True
for job in self.data.get('job', {}).values():
if jobs_filter and not matches(job['name'], jobs_filter):
logger.debug("Ignoring job {0}".format(job['name']))
continue
logger.debug("XMLifying job '{0}'".format(job['name']))
job = self.applyDefaults(job)
self.getXMLForJob(job)
for project in self.data.get('project', {}).values():
logger.debug("XMLifying project '{0}'".format(project['name']))
for jobspec in project.get('jobs', []):
if isinstance(jobspec, dict):
# Singleton dict containing dict of job-specific params
jobname, jobparams = jobspec.items()[0]
if not isinstance(jobparams, dict):
jobparams = {}
else:
jobname = jobspec
jobparams = {}
job = self.getJob(jobname)
if job:
# Just naming an existing defined job
continue
# see if it's a job group
group = self.getJobGroup(jobname)
if group:
for group_jobspec in group['jobs']:
if isinstance(group_jobspec, dict):
group_jobname, group_jobparams = \
group_jobspec.items()[0]
if not isinstance(group_jobparams, dict):
group_jobparams = {}
else:
group_jobname = group_jobspec
group_jobparams = {}
job = self.getJob(group_jobname)
if job:
continue
template = self.getJobTemplate(group_jobname)
# Allow a group to override parameters set by a project
d = {}
d.update(project)
d.update(jobparams)
d.update(group)
d.update(group_jobparams)
# Except name, since the group's name is not useful
d['name'] = project['name']
if template:
self.getXMLForTemplateJob(d, template, jobs_filter)
continue
# see if it's a template
template = self.getJobTemplate(jobname)
if template:
d = {}
d.update(project)
d.update(jobparams)
self.getXMLForTemplateJob(d, template, jobs_filter)
else:
raise JenkinsJobsException(
"Failed to find suitable "
"template named '{0}'".format(jobname))
def execute(options, config):
logger.debug("Config: {0}".format(config))
# check the ignore_cache setting: first from command line,
# if not present check from ini file
ignore_cache = False
if options.ignore_cache:
ignore_cache = options.ignore_cache
elif config.has_option('jenkins', 'ignore_cache'):
logging.warn('ignore_cache option should be moved to the [job_builder]'
' section in the config file, the one specified in the '
'[jenkins] section will be ignored in the future')
ignore_cache = config.getboolean('jenkins', 'ignore_cache')
elif config.has_option('job_builder', 'ignore_cache'):
ignore_cache = config.getboolean('job_builder', 'ignore_cache')
# Jenkins supports access as an anonymous user, which can be used to
# ensure read-only behaviour when querying the version of plugins
# installed for test mode to generate XML output matching what will be
# uploaded. To enable must pass 'None' as the value for user and password
# to python-jenkins
#
# catching 'TypeError' is a workaround for python 2.6 interpolation error
# https://bugs.launchpad.net/openstack-ci/+bug/1259631
try:
user = config.get('jenkins', 'user')
except (TypeError, configparser.NoOptionError):
user = None
try:
password = config.get('jenkins', 'password')
except (TypeError, configparser.NoOptionError):
password = None
# Inform the user as to what is likely to happen, as they may specify
# a real jenkins instance in test mode to get the plugin info to check
# the XML generated.
if user is None and password is None:
logger.info("Will use anonymous access to Jenkins if needed.")
elif (user is not None and password is None) or (
user is None and password is not None):
raise JenkinsJobsException(
"Cannot authenticate to Jenkins with only one of User and "
"Password provided, please check your configuration."
)
# None -- no timeout, blocking mode; same as setblocking(True)
# 0.0 -- non-blocking mode; same as setblocking(False) <--- default
# > 0 -- timeout mode; operations time out after timeout seconds
# < 0 -- illegal; raises an exception
# to retain the default must use
# "timeout=jenkins_jobs.builder._DEFAULT_TIMEOUT" or not set timeout at
# all.
timeout = jenkins_jobs.builder._DEFAULT_TIMEOUT
try:
timeout = config.getfloat('jenkins', 'timeout')
except (ValueError):
raise JenkinsJobsException("Jenkins timeout config is invalid")
except (TypeError, configparser.NoOptionError):
pass
plugins_info = None
if getattr(options, 'plugins_info_path', None) is not None:
with io.open(options.plugins_info_path, 'r',
encoding='utf-8') as yaml_file:
plugins_info = yaml.load(yaml_file)
if not isinstance(plugins_info, list):
raise JenkinsJobsException("{0} must contain a Yaml list!"
.format(options.plugins_info_path))
elif (not options.conf or not
config.getboolean("jenkins", "query_plugins_info")):
logger.debug("Skipping plugin info retrieval")
plugins_info = {}
if options.allow_empty_variables is not None:
config.set('job_builder',
'allow_empty_variables',
str(options.allow_empty_variables))
builder = Builder(config.get('jenkins', 'url'),
user,
password,
config,
jenkins_timeout=timeout,
ignore_cache=ignore_cache,
flush_cache=options.flush_cache,
plugins_list=plugins_info)
if getattr(options, 'path', None):
if hasattr(options.path, 'read'):
logger.debug("Input file is stdin")
if options.path.isatty():
key = 'CTRL+Z' if platform.system() == 'Windows' else 'CTRL+D'
logger.warn(
"Reading configuration from STDIN. Press %s to end input.",
key)
else:
# take list of paths
options.path = options.path.split(os.pathsep)
do_recurse = (getattr(options, 'recursive', False) or
config.getboolean('job_builder', 'recursive'))
excludes = [e for elist in options.exclude
for e in elist.split(os.pathsep)] or \
config.get('job_builder', 'exclude').split(os.pathsep)
paths = []
for path in options.path:
if do_recurse and os.path.isdir(path):
paths.extend(recurse_path(path, excludes))
else:
paths.append(path)
options.path = paths
if options.command == 'delete':
for job in options.name:
builder.delete_job(job, options.path)
elif options.command == 'delete-all':
confirm('Sure you want to delete *ALL* jobs from Jenkins server?\n'
'(including those not managed by Jenkins Job Builder)')
logger.info("Deleting all jobs")
builder.delete_all_jobs()
elif options.command == 'update':
if options.n_workers < 0:
raise JenkinsJobsException(
'Number of workers must be equal or greater than 0')
logger.info("Updating jobs in {0} ({1})".format(
options.path, options.names))
jobs, num_updated_jobs = builder.update_jobs(
options.path, options.names,
n_workers=options.n_workers)
logger.info("Number of jobs updated: %d", num_updated_jobs)
if options.delete_old:
num_deleted_jobs = builder.delete_old_managed()
logger.info("Number of jobs deleted: %d", num_deleted_jobs)
elif options.command == 'test':
builder.update_jobs(options.path, options.name,
output=options.output_dir,
n_workers=1)
def execute(options, config):
logger.debug("Config: {0}".format(config))
# check the ignore_cache setting: first from command line,
# if not present check from ini file
ignore_cache = False
if options.ignore_cache:
ignore_cache = options.ignore_cache
elif config.has_option('jenkins', 'ignore_cache'):
logging.warn('ignore_cache option should be moved to the [job_builder]'
' section in the config file, the one specified in the '
'[jenkins] section will be ignored in the future')
ignore_cache = config.getboolean('jenkins', 'ignore_cache')
elif config.has_option('job_builder', 'ignore_cache'):
ignore_cache = config.getboolean('job_builder', 'ignore_cache')
# workaround for python 2.6 interpolation error
# https://bugs.launchpad.net/openstack-ci/+bug/1259631
try:
user = config.get('jenkins', 'user')
except (TypeError, configparser.NoOptionError):
user = None
try:
password = config.get('jenkins', 'password')
except (TypeError, configparser.NoOptionError):
password = None
plugins_info = None
if getattr(options, 'plugins_info_path', None) is not None:
with open(options.plugins_info_path, 'r') as yaml_file:
plugins_info = yaml.load(yaml_file)
if not isinstance(plugins_info, list):
raise JenkinsJobsException("{0} must contain a Yaml list!".format(
options.plugins_info_path))
elif (not options.conf
or not config.getboolean("jenkins", "query_plugins_info")):
logger.debug("Skipping plugin info retrieval")
plugins_info = {}
if options.allow_empty_variables is not None:
config.set('job_builder', 'allow_empty_variables',
str(options.allow_empty_variables))
builder = Builder(config.get('jenkins', 'url'),
user,
password,
config,
ignore_cache=ignore_cache,
flush_cache=options.flush_cache,
plugins_list=plugins_info)
if getattr(options, 'path', None):
if options.path == sys.stdin:
logger.debug("Input file is stdin")
if options.path.isatty():
key = 'CTRL+Z' if platform.system() == 'Windows' else 'CTRL+D'
logger.warn(
"Reading configuration from STDIN. Press %s to end input.",
key)
# take list of paths
options.path = options.path.split(os.pathsep)
do_recurse = (getattr(options, 'recursive', False)
or config.getboolean('job_builder', 'recursive'))
excludes = [e for elist in options.exclude
for e in elist.split(os.pathsep)] or \
config.get('job_builder', 'exclude').split(os.pathsep)
paths = []
for path in options.path:
if do_recurse and os.path.isdir(path):
paths.extend(recurse_path(path, excludes))
else:
paths.append(path)
options.path = paths
if options.command == 'delete':
if options.del_views:
for view in options.name:
builder.delete_view(view, options.path)
else:
for job in options.name:
builder.delete_job(job, options.path)
elif options.command == 'delete-all':
deleting_jobs = False
deleting_views = False
if options.only_jobs == options.only_views:
# if (delete-all --jobs --views) or (delete-all)
# Equivalent of {NOT(--jobs XOR --views)} only passes if both
# subparameters are true or both false
reach = 'jobs AND views'
deleting_jobs = True
deleting_views = True
elif options.only_jobs and not options.only_views:
reach = 'jobs'
deleting_jobs = True
elif options.only_views and not options.only_jobs:
reach = 'views'
deleting_views = True
confirm(
'Sure you want to delete *ALL* %s from Jenkins server?\n'
'(including those not managed by Jenkins Job Builder)', reach)
if deleting_jobs:
logger.info("Deleting all jobs")
builder.delete_all_jobs()
if deleting_views:
logger.info("Deleting all views")
builder.delete_all_views()
elif options.command == 'update':
logger.info("Updating jobs in {0} ({1})".format(
options.path, options.names))
jobs, views, num_updated_jobs, num_updated_views = builder.update_job(
options.path, options.names)
logger.info("%d jobs updated, %d views updated", num_updated_jobs,
num_updated_views)
if options.delete_old:
num_deleted_jobs = builder.delete_old_managed()
logger.info("Number of jobs deleted: %d", num_deleted_jobs)
elif options.command == 'test':
builder.update_job(options.path,
options.name,
output=options.output_dir)
def extended_choice(parser, xml_parent, data):
"""yaml: extended-choice
Creates an extended choice property where values can be read from a file
Requires the Jenkins `Extended Choice Parameter Plugin.
<https://wiki.jenkins-ci.org/display/JENKINS/
Extended+Choice+Parameter+plugin>`_
:arg string name: name of the property
:arg string description: description of the property (optional, default '')
:arg string property-file: location of property file to read from
(optional, default '')
:arg string property-key: key for the property-file (optional, default '')
:arg bool quote-value: whether to put quotes around the property
when passing to Jenkins (optional, default false)
:arg string visible-items: number of items to show in the list
(optional, default 5)
:arg string type: type of select (optional, default single-select)
:arg string value: comma separated list of values for the single select
or multi-select box (optional, default '')
:arg string default-value: used to set the initial selection of the
single-select or multi-select box (optional, default '')
:arg string default-property-file: location of property file when default
value needs to come from a property file (optional, default '')
:arg string default-property-key: key for the default property file
(optional, default '')
Example::
properties:
- extended-choice:
name: FOO
description: A foo property
property-file: /home/foo/property.prop
property-key: key
quote-value: true
visible-items: 10
type: multi-select
value: foo,bar,select
default-value: foo
default-property-file: /home/property.prop
default-property-key: fookey
"""
definition = XML.SubElement(xml_parent,
'hudson.model.ParametersDefinitionProperty')
definitions = XML.SubElement(definition, 'parameterDefinitions')
extended = XML.SubElement(definitions, 'com.cwctravel.hudson.plugins.'
'extended__choice__parameter.'
'ExtendedChoiceParameterDefinition')
XML.SubElement(extended, 'name').text = data['name']
XML.SubElement(extended, 'description').text = data.get('description', '')
XML.SubElement(extended, 'quoteValue').text = str(data.get('quote-value',
False)).lower()
XML.SubElement(extended, 'visibleItemCount').text = data.get(
'visible-items', '5')
choice = data.get('type', 'single-select')
choicedict = {'single-select': 'PT_SINGLE_SELECT',
'multi-select': 'PT_MULTI_SELECT',
'radio': 'PT_RADIO',
'checkbox': 'PT_CHECKBOX'}
if choice not in choicedict:
raise JenkinsJobsException("Type entered is not valid, must be one "
"of: single-select, multi-select, radio, "
"or checkbox")
XML.SubElement(extended, 'type').text = choicedict[choice]
XML.SubElement(extended, 'value').text = data.get('value', '')
XML.SubElement(extended, 'propertyFile').text = data.get('property-file',
'')
XML.SubElement(extended, 'propertyKey').text = data.get('property-key', '')
XML.SubElement(extended, 'defaultValue').text = data.get('default-value',
'')
XML.SubElement(extended, 'defaultPropertyFile').text = data.get(
'default-property-file', '')
XML.SubElement(extended, 'defaultPropertyKey').text = data.get(
'default-property-key', '')
def dispatch(self,
component_type,
xml_parent,
component,
template_data={}):
"""This is a method that you can call from your implementation of
Base.gen_xml or component. It allows modules to define a type
of component, and benefit from extensibility via Python
entry points and Jenkins Job Builder :ref:`Macros <macro>`.
:arg string component_type: the name of the component
(e.g., `builder`)
:arg YAMLParser parser: the global YAML Parser
:arg Element xml_parent: the parent XML element
:arg dict template_data: values that should be interpolated into
the component definition
See :py:class:`jenkins_jobs.modules.base.Base` for how to register
components of a module.
See the Publishers module for a simple example of how to use
this method.
"""
if component_type not in self.modules_by_component_type:
raise JenkinsJobsException("Unknown component type: "
"'{0}'.".format(component_type))
entry_point = self.modules_by_component_type[component_type]
component_list_type = self.get_component_list_type(entry_point)
if isinstance(component, dict):
# The component is a singleton dictionary of name: dict(args)
name, component_data = next(iter(component.items()))
if template_data or isinstance(component_data, Jinja2Loader):
# Template data contains values that should be interpolated
# into the component definition. To handle Jinja2 templates
# that don't contain any variables, we also deep format those.
try:
component_data = deep_format(
component_data,
template_data,
self.jjb_config.yamlparser["allow_empty_variables"],
)
except Exception:
logging.error(
"Failure formatting component ('%s') data '%s'",
name,
component_data,
)
raise
else:
# The component is a simple string name, eg "run-tests"
name = component
component_data = {}
# Look for a component function defined in an entry point
eps = self._entry_points_cache.get(component_list_type)
if eps is None:
logging.debug("Caching entrypoints for %s" % component_list_type)
module_eps = []
# auto build entry points by inferring from base component_types
mod = pkg_resources.EntryPoint("__all__",
entry_point.module_name,
dist=entry_point.dist)
Mod = mod.load()
func_eps = [
Mod.__dict__.get(a) for a in dir(Mod)
if isinstance(Mod.__dict__.get(a), types.FunctionType)
]
for func_ep in func_eps:
try:
# extract entry point based on docstring
name_line = func_ep.__doc__.split("\n")
if not name_line[0].startswith("yaml:"):
logger.debug("Ignoring '%s' as an entry point" %
name_line)
continue
ep_name = name_line[0].split(" ")[1]
except (AttributeError, IndexError):
# AttributeError by docstring not being defined as
# a string to have split called on it.
# IndexError raised by name_line not containing anything
# after the 'yaml:' string.
logger.debug("Not including func '%s' as an entry point" %
func_ep.__name__)
continue
module_eps.append(
pkg_resources.EntryPoint(
ep_name,
entry_point.module_name,
dist=entry_point.dist,
attrs=(func_ep.__name__, ),
))
logger.debug(
"Adding auto EP '%s=%s:%s'" %
(ep_name, entry_point.module_name, func_ep.__name__))
# load from explicitly defined entry points
module_eps.extend(
list(
pkg_resources.iter_entry_points(
group="jenkins_jobs.{0}".format(component_list_type))))
eps = {}
for module_ep in module_eps:
if module_ep.name in eps:
raise JenkinsJobsException(
"Duplicate entry point found for component type: "
"'{0}', '{0}',"
"name: '{1}'".format(component_type, name))
eps[module_ep.name] = module_ep.load()
# cache both sets of entry points
self._entry_points_cache[component_list_type] = eps
logger.debug("Cached entry point group %s = %s",
component_list_type, eps)
# check for macro first
component = self.parser_data.get(component_type, {}).get(name)
if component:
if name in eps and name not in self.masked_warned:
self.masked_warned[name] = True
logger.warning("You have a macro ('%s') defined for '%s' "
"component type that is masking an inbuilt "
"definition" % (name, component_type))
for b in component[component_list_type]:
# Pass component_data in as template data to this function
# so that if the macro is invoked with arguments,
# the arguments are interpolated into the real defn.
self.dispatch(component_type, xml_parent, b, component_data)
elif name in eps:
func = eps[name]
func(self, xml_parent, component_data)
else:
raise JenkinsJobsException("Unknown entry point or macro '{0}' "
"for component type: '{1}'.".format(
name, component_type))
def gerrit_handle_legacy_configuration(data):
hyphenizer = re.compile("[A-Z]")
def hyphenize(attr):
"""Convert strings like triggerOn to trigger-on.
"""
return hyphenizer.sub(lambda x: "-%s" % x.group(0).lower(), attr)
def convert_dict(d, old_keys):
for old_key in old_keys:
if old_key in d:
new_key = hyphenize(old_key)
logger.warn(
"'%s' is deprecated and will be removed after "
"1.0.0, please use '%s' instead", old_key, new_key)
d[new_key] = d[old_key]
del d[old_key]
convert_dict(data, [
'triggerOnPatchsetUploadedEvent',
'triggerOnChangeAbandonedEvent',
'triggerOnChangeMergedEvent',
'triggerOnChangeRestoredEvent',
'triggerOnCommentAddedEvent',
'triggerOnDraftPublishedEvent',
'triggerOnRefUpdatedEvent',
'triggerApprovalCategory',
'triggerApprovalValue',
'overrideVotes',
'gerritBuildSuccessfulVerifiedValue',
'gerritBuildFailedVerifiedValue',
'failureMessage',
'skipVote',
])
for project in data['projects']:
convert_dict(project, [
'projectCompareType',
'projectPattern',
'branchCompareType',
'branchPattern',
])
old_format_events = OrderedDict(
(key, should_register) for key, should_register in six.iteritems(data)
if key.startswith('trigger-on-'))
trigger_on = data.setdefault('trigger-on', [])
if old_format_events:
logger.warn(
"The events: %s; which you used is/are deprecated. "
"Please use 'trigger-on' instead.", ', '.join(old_format_events))
if old_format_events and trigger_on:
raise JenkinsJobsException(
'Both, the new format (trigger-on) and old format (trigger-on-*) '
'gerrit events format found. Please use either the new or the old '
'format of trigger events definition.')
trigger_on.extend(
event_name[len('trigger-on-'):]
for event_name, should_register in six.iteritems(old_format_events)
if should_register)
for idx, event in enumerate(trigger_on):
if event == 'comment-added-event':
trigger_on[idx] = events = OrderedDict()
events['comment-added-event'] = OrderedDict(
(('approval-category', data['trigger-approval-category']),
('approval-value', data['trigger-approval-value'])))
def deep_format(obj, paramdict, allow_empty=False):
"""Deep format configuration.
Apply the paramdict via str.format() to all string objects found within
the supplied obj. Lists and dicts are traversed recursively.
"""
# YAML serialisation was originally used to achieve this, but that places
# limitations on the values in paramdict - the post-format result must
# still be valid YAML (so substituting-in a string containing quotes, for
# example, is problematic).
if hasattr(obj, "format"):
try:
ret = CustomFormatter(allow_empty).format(obj, **paramdict)
except KeyError as exc:
missing_key = exc.args[0]
desc = "%s parameter missing to format %s\nGiven:\n%s" % (
missing_key,
obj,
pformat(paramdict),
)
raise JenkinsJobsException(desc)
except Exception:
logging.error(
"Problem formatting with args:\nallow_empty:"
"%s\nobj: %s\nparamdict: %s" % (allow_empty, obj, paramdict)
)
raise
elif isinstance(obj, list):
ret = type(obj)()
for item in obj:
ret.append(deep_format(item, paramdict, allow_empty))
elif isinstance(obj, dict):
ret = type(obj)()
for item in obj:
try:
ret[deep_format(item, paramdict, allow_empty)] = deep_format(
obj[item], paramdict, allow_empty
)
except KeyError as exc:
missing_key = exc.args[0]
desc = "%s parameter missing to format %s\nGiven:\n%s" % (
missing_key,
obj,
pformat(paramdict),
)
raise JenkinsJobsException(desc)
except Exception:
logging.error(
"Problem formatting with args:\nallow_empty:"
"%s\nobj: %s\nparamdict: %s" % (allow_empty, obj, paramdict)
)
raise
else:
ret = obj
if isinstance(ret, CustomLoader):
# If we have a CustomLoader here, we've lazily-loaded a template
# or rendered a template to a piece of YAML;
# attempt to format it.
ret = deep_format(
ret.get_object_to_format(), paramdict, allow_empty=allow_empty
)
return ret
def execute(options, config):
logger.debug("Config: {0}".format(config))
# check the ignore_cache setting: first from command line,
# if not present check from ini file
ignore_cache = False
if options.ignore_cache:
ignore_cache = options.ignore_cache
elif config.has_option('jenkins', 'ignore_cache'):
logging.warn('ignore_cache option should be moved to the [job_builder]'
' section in the config file, the one specified in the '
'[jenkins] section will be ignored in the future')
ignore_cache = config.getboolean('jenkins', 'ignore_cache')
elif config.has_option('job_builder', 'ignore_cache'):
ignore_cache = config.getboolean('job_builder', 'ignore_cache')
# workaround for python 2.6 interpolation error
# https://bugs.launchpad.net/openstack-ci/+bug/1259631
try:
user = config.get('jenkins', 'user')
except (TypeError, configparser.NoOptionError):
user = None
try:
password = config.get('jenkins', 'password')
except (TypeError, configparser.NoOptionError):
password = None
plugins_info = None
if getattr(options, 'plugins_info_path', None) is not None:
with open(options.plugins_info_path, 'r') as yaml_file:
plugins_info = yaml.load(yaml_file)
if not isinstance(plugins_info, list):
raise JenkinsJobsException("{0} must contain a Yaml list!".format(
options.plugins_info_path))
builder = Builder(config.get('jenkins', 'url'),
user,
password,
config,
ignore_cache=ignore_cache,
flush_cache=options.flush_cache,
plugins_list=plugins_info)
if getattr(options, 'path', None):
if options.path == sys.stdin:
logger.debug("Input file is stdin")
if options.path.isatty():
key = 'CTRL+Z' if platform.system() == 'Windows' else 'CTRL+D'
logger.warn(
"Reading configuration from STDIN. Press %s to end input.",
key)
# take list of paths
options.path = options.path.split(os.pathsep)
do_recurse = (getattr(options, 'recursive', False)
or config.getboolean('job_builder', 'recursive'))
paths = []
for path in options.path:
if do_recurse and os.path.isdir(path):
paths.extend(recurse_path(path))
else:
paths.append(path)
options.path = paths
if options.command == 'delete':
for job in options.name:
builder.delete_job(job, options.path)
elif options.command == 'delete-all':
confirm('Sure you want to delete *ALL* jobs from Jenkins server?\n'
'(including those not managed by Jenkins Job Builder)')
logger.info("Deleting all jobs")
builder.delete_all_jobs()
elif options.command == 'update':
logger.info("Updating jobs in {0} ({1})".format(
options.path, options.names))
jobs = builder.update_job(options.path, options.names)
if options.delete_old:
builder.delete_old_managed(keep=[x.name for x in jobs])
elif options.command == 'test':
builder.update_job(options.path,
options.name,
output=options.output_dir)
def build_gerrit_triggers(xml_parent, data):
available_simple_triggers = {
'change-abandoned-event': 'PluginChangeAbandonedEvent',
'change-merged-event': 'PluginChangeMergedEvent',
'change-restored-event': 'PluginChangeRestoredEvent',
'draft-published-event': 'PluginDraftPublishedEvent',
'patchset-uploaded-event': 'PluginPatchsetCreatedEvent',
'patchset-created-event': 'PluginPatchsetCreatedEvent',
'ref-updated-event': 'PluginRefUpdatedEvent',
}
tag_namespace = 'com.sonyericsson.hudson.plugins.gerrit.trigger.' \
'hudsontrigger.events'
trigger_on_events = XML.SubElement(xml_parent, 'triggerOnEvents')
for event in data.get('trigger-on', []):
if isinstance(event, six.string_types):
tag_name = available_simple_triggers.get(event)
if event == 'patchset-uploaded-event':
logger.warn(
"'%s' is deprecated. Use 'patchset-created-event' "
"format instead.", event)
if not tag_name:
known = ', '.join(
available_simple_triggers.keys() +
['comment-added-event', 'comment-added-contains-event'])
msg = ("The event '%s' under 'trigger-on' is not one of the "
"known: %s.") % (event, known)
raise JenkinsJobsException(msg)
XML.SubElement(trigger_on_events,
'%s.%s' % (tag_namespace, tag_name))
else:
if 'patchset-created-event' in event.keys():
pce = event['patchset-created-event']
pc = XML.SubElement(
trigger_on_events,
'%s.%s' % (tag_namespace, 'PluginPatchsetCreatedEvent'))
XML.SubElement(pc, 'excludeDrafts').text = str(
pce.get('exclude-drafts', False)).lower()
XML.SubElement(pc, 'excludeTrivialRebase').text = str(
pce.get('exclude-trivial-rebase', False)).lower()
XML.SubElement(pc, 'excludeNoCodeChange').text = str(
pce.get('exclude-no-code-change', False)).lower()
if 'comment-added-event' in event.keys():
comment_added_event = event['comment-added-event']
cadded = XML.SubElement(
trigger_on_events,
'%s.%s' % (tag_namespace, 'PluginCommentAddedEvent'))
XML.SubElement(cadded, 'verdictCategory').text = \
comment_added_event['approval-category']
XML.SubElement(
cadded,
'commentAddedTriggerApprovalValue').text = \
str(comment_added_event['approval-value'])
if 'comment-added-contains-event' in event.keys():
comment_added_event = event['comment-added-contains-event']
caddedc = XML.SubElement(
trigger_on_events, '%s.%s' %
(tag_namespace, 'PluginCommentAddedContainsEvent'))
XML.SubElement(caddedc, 'commentAddedCommentContains').text = \
comment_added_event['comment-contains-value']
def build_trends_publisher(plugin_name, xml_element, data):
"""Helper to create various trend publishers.
"""
def append_thresholds(element, data, only_totals):
"""Appends the status thresholds.
"""
for status in ["unstable", "failed"]:
status_data = data.get(status, {})
limits = [
("total-all", "TotalAll"),
("total-high", "TotalHigh"),
("total-normal", "TotalNormal"),
("total-low", "TotalLow"),
]
if only_totals is False:
limits.extend([
("new-all", "NewAll"),
("new-high", "NewHigh"),
("new-normal", "NewNormal"),
("new-low", "NewLow"),
])
for key, tag_suffix in limits:
tag_name = status + tag_suffix
XML.SubElement(element,
tag_name).text = str(status_data.get(key, ""))
# Tuples containing: setting name, tag name, default value
settings = [
("healthy", "healthy", ""),
("unhealthy", "unHealthy", ""),
("health-threshold", "thresholdLimit", "low"),
("plugin-name", "pluginName", plugin_name),
("default-encoding", "defaultEncoding", ""),
("can-run-on-failed", "canRunOnFailed", False),
("use-stable-build-as-reference", "useStableBuildAsReference", False),
("use-previous-build-as-reference", "usePreviousBuildAsReference",
False),
("use-delta-values", "useDeltaValues", False),
("thresholds", "thresholds", {}),
("should-detect-modules", "shouldDetectModules", False),
("dont-compute-new", "dontComputeNew", True),
("do-not-resolve-relative-paths", "doNotResolveRelativePaths", False),
("pattern", "pattern", ""),
]
thresholds = ["low", "normal", "high"]
for key, tag_name, default in settings:
xml_config = XML.SubElement(xml_element, tag_name)
config_value = data.get(key, default)
if key == "thresholds":
append_thresholds(xml_config, config_value,
data.get("dont-compute-new", True))
elif key == "health-threshold" and config_value not in thresholds:
raise JenkinsJobsException("health-threshold must be one of %s" %
", ".join(thresholds))
else:
if isinstance(default, bool):
xml_config.text = str(config_value).lower()
else:
xml_config.text = str(config_value)
def copyartifact(parser, xml_parent, data):
"""yaml: copyartifact
Copy artifact from another project. Requires the Jenkins `Copy Artifact
plugin.
<https://wiki.jenkins-ci.org/display/JENKINS/Copy+Artifact+Plugin>`_
:arg str project: Project to copy from
:arg str filter: what files to copy
:arg str target: Target base directory for copy, blank means use workspace
:arg bool flatten: Flatten directories (default: false)
:arg bool optional: If the artifact is missing (for any reason) and
optional is true, the build won't fail because of this builder
(default: false)
:arg str which-build: which build to get artifacts from
(optional, default last-successful)
:arg str build-number: specifies the build number to get when
when specific-build is specified as which-build
:arg str permalink: specifies the permalink to get when
permalink is specified as which-build
:arg bool stable: specifies to get only last stable build when
last-successful is specified as which-build
:arg bool fallback-to-last-successful: specifies to fallback to
last successful build when upstream-build is specified as which-build
:arg string param: specifies to use a build parameter to get the build when
build-param is specified as which-build
:arg string parameter-filters: Filter matching jobs based on these
parameters (optional)
:which-build values:
* **last-successful**
* **specific-build**
* **last-saved**
* **upstream-build**
* **permalink**
* **workspace-latest**
* **build-param**
:permalink values:
* **last**
* **last-stable**
* **last-successful**
* **last-failed**
* **last-unstable**
* **last-unsuccessful**
Example::
builders:
- copyartifact:
project: foo
filter: *.tar.gz
target: /home/foo
which-build: specific-build
build-number: 123
optional: true
flatten: true
parameter-filters: PUBLISH=true
"""
t = XML.SubElement(xml_parent, 'hudson.plugins.copyartifact.CopyArtifact')
# Warning: this only works with copy artifact version 1.26+,
# for copy artifact version 1.25- the 'projectName' element needs
# to be used instead of 'project'
XML.SubElement(t, 'project').text = data["project"]
XML.SubElement(t, 'filter').text = data.get("filter", "")
XML.SubElement(t, 'target').text = data.get("target", "")
flatten = data.get("flatten", False)
XML.SubElement(t, 'flatten').text = str(flatten).lower()
optional = data.get('optional', False)
XML.SubElement(t, 'optional').text = str(optional).lower()
XML.SubElement(t, 'parameters').text = data.get("parameter-filters", "")
select = data.get('which-build', 'last-successful')
selectdict = {
'last-successful': 'StatusBuildSelector',
'specific-build': 'SpecificBuildSelector',
'last-saved': 'SavedBuildSelector',
'upstream-build': 'TriggeredBuildSelector',
'permalink': 'PermalinkBuildSelector',
'workspace-latest': 'WorkspaceSelector',
'build-param': 'ParameterizedBuildSelector'
}
if select not in selectdict:
raise JenkinsJobsException("which-build entered is not valid must be "
"one of: last-successful, specific-build, "
"last-saved, upstream-build, permalink, "
"workspace-latest, or build-param")
permalink = data.get('permalink', 'last')
permalinkdict = {
'last': 'lastBuild',
'last-stable': 'lastStableBuild',
'last-successful': 'lastSuccessfulBuild',
'last-failed': 'lastFailedBuild',
'last-unstable': 'lastUnstableBuild',
'last-unsuccessful': 'lastUnsuccessfulBuild'
}
if permalink not in permalinkdict:
raise JenkinsJobsException("permalink entered is not valid must be "
"one of: last, last-stable, "
"last-successful, last-failed, "
"last-unstable, or last-unsuccessful")
selector = XML.SubElement(
t, 'selector',
{'class': 'hudson.plugins.copyartifact.' + selectdict[select]})
if select == 'specific-build':
XML.SubElement(selector, 'buildNumber').text = data['build-number']
if select == 'last-successful':
XML.SubElement(selector,
'stable').text = str(data.get('stable', False)).lower()
if select == 'upstream-build':
XML.SubElement(selector, 'fallbackToLastSuccessful').text = str(
data.get('fallback-to-last-successful', False)).lower()
if select == 'permalink':
XML.SubElement(selector, 'id').text = permalinkdict[permalink]
if select == 'build-param':
XML.SubElement(selector, 'parameterName').text = data['param']
def dispatch(self,
component_type,
parser,
xml_parent,
component,
template_data={}):
"""This is a method that you can call from your implementation of
Base.gen_xml or component. It allows modules to define a type
of component, and benefit from extensibility via Python
entry points and Jenkins Job Builder :ref:`Macros <macro>`.
:arg string component_type: the name of the component
(e.g., `builder`)
:arg YAMLParser parser: the global YAML Parser
:arg Element xml_parent: the parent XML element
:arg dict template_data: values that should be interpolated into
the component definition
See :py:class:`jenkins_jobs.modules.base.Base` for how to register
components of a module.
See the Publishers module for a simple example of how to use
this method.
"""
if component_type not in self.modules_by_component_type:
raise JenkinsJobsException("Unknown component type: "
"'{0}'.".format(component_type))
component_list_type = self.modules_by_component_type[component_type] \
.component_list_type
if isinstance(component, dict):
# The component is a singleton dictionary of name: dict(args)
name, component_data = next(iter(component.items()))
if template_data:
# Template data contains values that should be interpolated
# into the component definition
allow_empty_variables = self.global_config \
and self.global_config.has_section('job_builder') \
and self.global_config.has_option(
'job_builder', 'allow_empty_variables') \
and self.global_config.getboolean(
'job_builder', 'allow_empty_variables')
component_data = deep_format(component_data, template_data,
allow_empty_variables)
else:
# The component is a simple string name, eg "run-tests"
name = component
component_data = {}
# Look for a component function defined in an entry point
eps = ModuleRegistry.entry_points_cache.get(component_list_type)
if eps is None:
module_eps = list(
pkg_resources.iter_entry_points(
group='jenkins_jobs.{0}'.format(component_list_type)))
eps = {}
for module_ep in module_eps:
if module_ep.name in eps:
raise JenkinsJobsException(
"Duplicate entry point found for component type: "
"'{0}', '{0}',"
"name: '{1}'".format(component_type, name))
eps[module_ep.name] = module_ep
ModuleRegistry.entry_points_cache[component_list_type] = eps
logger.debug("Cached entry point group %s = %s",
component_list_type, eps)
if name in eps:
func = eps[name].load()
func(parser, xml_parent, component_data)
else:
# Otherwise, see if it's defined as a macro
component = parser.data.get(component_type, {}).get(name)
if component:
for b in component[component_list_type]:
# Pass component_data in as template data to this function
# so that if the macro is invoked with arguments,
# the arguments are interpolated into the real defn.
self.dispatch(component_type, parser, xml_parent, b,
component_data)
else:
raise JenkinsJobsException(
"Unknown entry point or macro '{0}'"
" for component type: '{1}'.".format(name, component_type))
def git(parser, xml_parent, data):
"""yaml: git
Specifies the git SCM repository for this job.
Requires the Jenkins `Git Plugin.
<https://wiki.jenkins-ci.org/display/JENKINS/Git+Plugin>`_
:arg str url: URL of the git repository
:arg str credentials-id: ID of credentials to use to connect (optional)
:arg str refspec: refspec to fetch (default '+refs/heads/\*:refs/remotes/\
remoteName/\*')
:arg str name: name to fetch (default 'origin')
:arg list(str) remotes: list of remotes to set up (optional, only needed if
multiple remotes need to be set up)
:Remote: * **url** (`string`) - url of remote repo
* **refspec** (`string`) - refspec to fetch (optional)
* **credentials-id** - ID of credentials to use to connect
(optional)
:arg list(str) branches: list of branch specifiers to build (default '**')
:arg list(str) excluded-users: list of users to ignore revisions from
when polling for changes. (if polling is enabled, optional)
:arg list(str) included-regions: list of file/folders to include (optional)
:arg list(str) excluded-regions: list of file/folders to exclude (optional)
:arg str local-branch: Checkout/merge to local branch (optional)
:arg dict merge:
:merge:
* **remote** (`string`) - name of repo that contains branch to
merge to (default 'origin')
* **branch** (`string`) - name of the branch to merge to
:arg str basedir: location relative to the workspace root to clone to
(default: workspace)
:arg bool skip-tag: Skip tagging (default false)
:arg bool shallow-clone: Perform shallow clone (default false)
:arg bool prune: Prune remote branches (default false)
:arg bool clean: Clean after checkout (default false)
:arg bool fastpoll: Use fast remote polling (default false)
:arg bool disable-submodules: Disable submodules (default false)
:arg bool recursive-submodules: Recursively update submodules (default
false)
:arg bool use-author: Use author rather than committer in Jenkin's build
changeset (default false)
:arg str git-tool: The name of the Git installation to use (default
'Default')
:arg str reference-repo: Path of the reference repo to use during clone
(optional)
:arg str scm-name: The unique scm name for this Git SCM (optional)
:arg bool wipe-workspace: Wipe out workspace before build (default true)
:arg bool ignore-notify: Ignore notifyCommit URL accesses (default false)
:arg str browser: what repository browser to use (default '(Auto)')
:arg str browser-url: url for the repository browser (required if browser
is not '(Auto)', no default)
:arg str browser-version: version of the repository browser (GitLab only,
default '0.0')
:arg str project-name: project name in Gitblit and ViewGit repobrowser
(optional)
:arg str choosing-strategy: Jenkins class for selecting what to build
(default 'default')
:arg str git-config-name: Configure name for Git clone (optional)
:arg str git-config-email: Configure email for Git clone (optional)
:arg str timeout: Timeout for git commands in minutes (optional)
:browser values:
:auto:
:bitbucketweb:
:cgit:
:fisheye:
:gitblit:
:githubweb:
:gitlab:
:gitoriousweb:
:gitweb:
:redmineweb:
:stash:
:viewgit:
:choosing-strategy values:
:default:
:inverse:
:gerrit:
Example:
.. literalinclude:: /../../tests/scm/fixtures/git001.yaml
"""
# XXX somebody should write the docs for those with option name =
# None so we have a sensible name/key for it.
mapping = [
# option, xml name, default value (text), attributes (hard coded)
("disable-submodules", 'disableSubmodules', False),
("recursive-submodules", 'recursiveSubmodules', False),
(None, 'doGenerateSubmoduleConfigurations', False),
("use-author", 'authorOrCommitter', False),
("clean", 'clean', False),
("wipe-workspace", 'wipeOutWorkspace', True),
("prune", 'pruneBranches', False),
("fastpoll", 'remotePoll', False),
("git-tool", 'gitTool', "Default"),
(None, 'submoduleCfg', '', {
'class': 'list'
}),
('basedir', 'relativeTargetDir', ''),
('reference-repo', 'reference', ''),
("git-config-name", 'gitConfigName', ''),
("git-config-email", 'gitConfigEmail', ''),
('skip-tag', 'skipTag', False),
('scm-name', 'scmName', ''),
("shallow-clone", "useShallowClone", False),
("ignore-notify", "ignoreNotifyCommit", False),
]
choosing_strategies = {
'default':
'hudson.plugins.git.util.DefaultBuildChooser',
'gerrit': ('com.sonyericsson.hudson.plugins.'
'gerrit.trigger.hudsontrigger.GerritTriggerBuildChooser'),
'inverse':
'hudson.plugins.git.util.InverseBuildChooser',
}
scm = XML.SubElement(xml_parent, 'scm',
{'class': 'hudson.plugins.git.GitSCM'})
XML.SubElement(scm, 'configVersion').text = '2'
user = XML.SubElement(scm, 'userRemoteConfigs')
if 'remotes' not in data:
data['remotes'] = [{data.get('name', 'origin'): data.copy()}]
for remoteData in data['remotes']:
huser = XML.SubElement(user, 'hudson.plugins.git.UserRemoteConfig')
remoteName = next(iter(remoteData.keys()))
XML.SubElement(huser, 'name').text = remoteName
remoteParams = next(iter(remoteData.values()))
if 'refspec' in remoteParams:
refspec = remoteParams['refspec']
else:
refspec = '+refs/heads/*:refs/remotes/' + remoteName + '/*'
XML.SubElement(huser, 'refspec').text = refspec
if 'url' in remoteParams:
remoteURL = remoteParams['url']
else:
raise JenkinsJobsException('Must specify a url for git remote \"' +
remoteName + '"')
XML.SubElement(huser, 'url').text = remoteURL
if 'credentials-id' in remoteParams:
credentialsId = remoteParams['credentials-id']
XML.SubElement(huser, 'credentialsId').text = credentialsId
xml_branches = XML.SubElement(scm, 'branches')
branches = data.get('branches', ['**'])
for branch in branches:
bspec = XML.SubElement(xml_branches, 'hudson.plugins.git.BranchSpec')
XML.SubElement(bspec, 'name').text = branch
excluded_users = '\n'.join(data.get('excluded-users', []))
XML.SubElement(scm, 'excludedUsers').text = excluded_users
if 'included-regions' in data:
include_string = '\n'.join(data['included-regions'])
XML.SubElement(scm, 'includedRegions').text = include_string
if 'excluded-regions' in data:
exclude_string = '\n'.join(data['excluded-regions'])
XML.SubElement(scm, 'excludedRegions').text = exclude_string
if 'merge' in data:
merge = data['merge']
name = merge.get('remote', 'origin')
branch = merge['branch']
urc = XML.SubElement(scm, 'userMergeOptions')
XML.SubElement(urc, 'mergeRemote').text = name
XML.SubElement(urc, 'mergeTarget').text = branch
try:
choosing_strategy = choosing_strategies[data.get(
'choosing-strategy', 'default')]
except KeyError:
raise ValueError('Invalid choosing-strategy %r' %
data.get('choosing-strategy'))
XML.SubElement(scm, 'buildChooser', {'class': choosing_strategy})
for elem in mapping:
(optname, xmlname, val) = elem[:3]
attrs = {}
if len(elem) >= 4:
attrs = elem[3]
xe = XML.SubElement(scm, xmlname, attrs)
if optname and optname in data:
val = data[optname]
if type(val) == bool:
xe.text = str(val).lower()
else:
xe.text = val
if 'local-branch' in data:
XML.SubElement(scm, 'localBranch').text = data['local-branch']
if 'timeout' in data:
ext = XML.SubElement(scm, 'extensions')
co = XML.SubElement(
ext, 'hudson.plugins.git.extensions.impl.'
'CheckoutOption')
XML.SubElement(co, 'timeout').text = str(data['timeout'])
browser = data.get('browser', 'auto')
browserdict = {
'auto': 'auto',
'bitbucketweb': 'BitbucketWeb',
'cgit': 'CGit',
'fisheye': 'FisheyeGitRepositoryBrowser',
'gitblit': 'GitBlitRepositoryBrowser',
'githubweb': 'GithubWeb',
'gitlab': 'GitLab',
'gitoriousweb': 'GitoriousWeb',
'gitweb': 'GitWeb',
'redmineweb': 'RedmineWeb',
'stash': 'Stash',
'viewgit': 'ViewGitWeb'
}
if browser not in browserdict:
valid = sorted(browserdict.keys())
raise JenkinsJobsException("Browser entered is not valid must be one "
"of: %s or %s." %
(", ".join(valid[:-1]), valid[-1]))
if browser != 'auto':
bc = XML.SubElement(
scm, 'browser',
{'class': 'hudson.plugins.git.browser.' + browserdict[browser]})
XML.SubElement(bc, 'url').text = data['browser-url']
if browser in ['gitblit', 'viewgit']:
XML.SubElement(bc, 'projectName').text = str(
data.get('project-name', ''))
if browser == 'gitlab':
XML.SubElement(bc, 'version').text = str(
data.get('browser-version', '0.0'))
def pollurl(parser, xml_parent, data):
"""yaml: pollurl
Trigger when the HTTP response from a URL changes.
Requires the Jenkins `URLTrigger Plugin.
<https://wiki.jenkins-ci.org/display/JENKINS/URLTrigger+Plugin>`_
:arg string cron: cron syntax of when to run (default '')
:arg string polling-node: Restrict where the polling should run.
(optional)
:arg list urls: List of URLs to monitor
:URL: * **url** (`str`) -- URL to monitor for changes (required)
* **proxy** (`bool`) -- Activate the Jenkins proxy (default false)
* **timeout** (`int`) -- Connect/read timeout in seconds
(default 300)
* **username** (`string`) -- User name for basic authentication
(optional)
* **password** (`string`) -- Password for basic authentication
(optional)
* **check-status** (`int`) -- Check for a specific HTTP status
code (optional)
* **check-etag** (`bool`) -- Check the HTTP ETag for changes
(default false)
* **check-date** (`bool`) -- Check the last modification date of
the URL (default false)
* **check-content** (`list`) -- List of content type changes to
monitor
:Content Type: * **simple** (`bool`) -- Trigger on any change to
the content of the URL (default false)
* **json** (`list`) -- Trigger on any change to
the listed JSON paths
* **text** (`list`) -- Trigger on any change to
the listed regular expressions
* **xml** (`list`) -- Trigger on any change to
the listed XPath expressions
Example:
.. literalinclude:: /../../tests/triggers/fixtures/pollurl001.yaml
"""
valid_content_types = {
'simple': ['Simple', '', '', []],
'json': ['JSON', 'jsonPaths', 'jsonPath', None],
'text': ['TEXT', 'regExElements', 'regEx', None],
'xml': ['XML', 'xPaths', 'xPath', None]
}
urltrig = XML.SubElement(xml_parent,
'org.jenkinsci.plugins.urltrigger.URLTrigger')
node = data.get('polling-node')
XML.SubElement(urltrig, 'spec').text = data.get('cron', '')
XML.SubElement(urltrig, 'labelRestriction').text = str(bool(node)).lower()
if node:
XML.SubElement(urltrig, 'triggerLabel').text = node
entries = XML.SubElement(urltrig, 'entries')
urls = data.get('urls', [])
if not urls:
raise JenkinsJobsException('At least one url must be provided')
for url in urls:
entry = XML.SubElement(entries,
'org.jenkinsci.plugins.urltrigger.'
'URLTriggerEntry')
XML.SubElement(entry, 'url').text = url['url']
XML.SubElement(entry, 'proxyActivated').text = \
str(url.get('proxy', False)).lower()
if 'username' in url:
XML.SubElement(entry, 'username').text = url['username']
if 'password' in url:
XML.SubElement(entry, 'password').text = url['password']
if 'check-status' in url:
XML.SubElement(entry, 'checkStatus').text = 'true'
XML.SubElement(entry, 'statusCode').text = \
str(url.get('check-status'))
else:
XML.SubElement(entry, 'checkStatus').text = 'false'
XML.SubElement(entry, 'statusCode').text = '200'
XML.SubElement(entry, 'timeout').text = \
str(url.get('timeout', 300))
XML.SubElement(entry, 'checkETag').text = \
str(url.get('check-etag', False)).lower()
XML.SubElement(entry, 'checkLastModificationDate').text = \
str(url.get('check-date', False)).lower()
check_content = url.get('check-content', [])
XML.SubElement(entry, 'inspectingContent').text = \
str(bool(check_content)).lower()
content_types = XML.SubElement(entry, 'contentTypes')
for entry in check_content:
type_name = entry.keys()[0]
if type_name not in valid_content_types:
raise JenkinsJobsException('check-content must be one of : %s'
% ', '.join(valid_content_types.
keys()))
content_type = valid_content_types.get(type_name)
if entry[type_name]:
sub_entries = content_type[3]
if sub_entries is None:
sub_entries = entry[type_name]
build_pollurl_content_type(content_types,
sub_entries,
*content_type[0:3])
def svn(parser, xml_parent, data):
"""yaml: svn
Specifies the svn SCM repository for this job.
:arg str url: URL of the svn repository
:arg str basedir: location relative to the workspace root to checkout to
(default '.')
:arg str credentials-id: optional argument to specify the ID of credentials
to use
:arg str workspaceupdater: optional argument to specify
how to update the workspace (default wipeworkspace)
:arg list(str) excluded-users: list of users to ignore revisions from
when polling for changes (if polling is enabled; parameter is optional)
:arg list(str) included-regions: list of file/folders to include
(optional)
:arg list(str) excluded-regions: list of file/folders to exclude (optional)
:arg list(str) excluded-commit-messages: list of commit messages to exclude
(optional)
:arg str exclusion-revprop-name: revision svn-property to ignore (optional)
:arg bool ignore-property-changes-on-directories: ignore svn-property only
changes of directories (default false)
:arg bool filter-changelog: If set Jenkins will apply the same inclusion
and exclusion patterns for displaying changelog entries as it does for
polling for changes (default false)
:arg list repos: list of repositories to checkout (optional)
:Repo: * **url** (`str`) -- URL for the repository
* **basedir** (`str`) -- Location relative to the workspace
root to checkout to (default '.')
* **credentials-id** - optional ID of credentials to use
:workspaceupdater values:
:wipeworkspace: - deletes the workspace before checking out
:revertupdate: - do an svn revert then an svn update
:emulateclean: - delete unversioned/ignored files then update
:update: - do an svn update as much as possible
Multiple repos example:
.. literalinclude:: /../../tests/scm/fixtures/svn-multiple-repos-001.yaml
Advanced commit filtering example:
.. literalinclude:: /../../tests/scm/fixtures/svn-regions-001.yaml
"""
scm = XML.SubElement(xml_parent, 'scm',
{'class': 'hudson.scm.SubversionSCM'})
locations = XML.SubElement(scm, 'locations')
if 'repos' in data:
repos = data['repos']
for repo in repos:
module = XML.SubElement(
locations, 'hudson.scm.SubversionSCM_-ModuleLocation')
XML.SubElement(module, 'remote').text = repo['url']
XML.SubElement(module, 'local').text = repo.get('basedir', '.')
if 'credentials-id' in repo:
XML.SubElement(module,
'credentialsId').text = repo['credentials-id']
elif 'url' in data:
module = XML.SubElement(locations,
'hudson.scm.SubversionSCM_-ModuleLocation')
XML.SubElement(module, 'remote').text = data['url']
XML.SubElement(module, 'local').text = data.get('basedir', '.')
if 'credentials-id' in data:
XML.SubElement(module,
'credentialsId').text = data['credentials-id']
else:
raise JenkinsJobsException("A top level url or repos list must exist")
updater = data.get('workspaceupdater', 'wipeworkspace')
if updater == 'wipeworkspace':
updaterclass = 'CheckoutUpdater'
elif updater == 'revertupdate':
updaterclass = 'UpdateWithRevertUpdater'
elif updater == 'emulateclean':
updaterclass = 'UpdateWithCleanUpdater'
elif updater == 'update':
updaterclass = 'UpdateUpdater'
XML.SubElement(scm, 'workspaceUpdater',
{'class': 'hudson.scm.subversion.' + updaterclass})
mapping = [
# option, xml name, default value
("excluded-regions", 'excludedRegions', []),
("included-regions", 'includedRegions', []),
("excluded-users", 'excludedUsers', []),
("exclusion-revprop-name", 'excludedRevprop', ''),
("excluded-commit-messages", 'excludedCommitMessages', []),
("ignore-property-changes-on-directories", 'ignoreDirPropChanges',
False),
("filter-changelog", 'filterChangelog', False),
]
for optname, xmlname, defvalue in mapping:
if isinstance(defvalue, list):
val = '\n'.join(data.get(optname, defvalue))
else:
val = data.get(optname, defvalue)
# Skip adding xml entry if default is empty and no value given
if not val and (defvalue in ['', []]):
continue
xe = XML.SubElement(scm, xmlname)
if isinstance(defvalue, bool):
xe.text = str(val).lower()
else:
xe.text = str(val)
def _setup(self):
config = self.config_parser
logger.debug("Config: {0}".format(config))
# check the ignore_cache setting
if config.has_option(self.section, 'ignore_cache'):
logging.warning("ignore_cache option should be moved to the "
"[job_builder] section in the config file, the "
"one specified in the [jenkins] section will be "
"ignored in the future")
self.ignore_cache = config.getboolean(self.section, 'ignore_cache')
elif config.has_option('job_builder', 'ignore_cache'):
self.ignore_cache = config.getboolean('job_builder',
'ignore_cache')
# check the flush_cache setting
if config.has_option('job_builder', 'flush_cache'):
self.flush_cache = config.getboolean('job_builder', 'flush_cache')
# Jenkins supports access as an anonymous user, which can be used to
# ensure read-only behaviour when querying the version of plugins
# installed for test mode to generate XML output matching what will be
# uploaded. To enable must pass 'None' as the value for user and
# password to python-jenkins
#
# catching 'TypeError' is a workaround for python 2.6 interpolation
# error
# https://bugs.launchpad.net/openstack-ci/+bug/1259631
try:
self.user = config.get(self.section, 'user')
except (TypeError, configparser.NoOptionError):
pass
try:
if config.has_option(self.section, 'password'):
self.password = config.get(self.section, 'password')
else:
self.password = getpass("Enter Jenkins config Password:"******"timeout=jenkins_jobs.builder._DEFAULT_TIMEOUT" or not set timeout at
# all.
try:
self.timeout = config.getfloat(self.section, 'timeout')
except (ValueError):
raise JenkinsJobsException("Jenkins timeout config is invalid")
except (TypeError, configparser.NoOptionError):
pass
if (config.has_option(self.section, 'query_plugins_info')
and not config.getboolean(self.section, "query_plugins_info")):
logger.debug("Skipping plugin info retrieval")
self.plugins_info = []
self.recursive = config.getboolean('job_builder', 'recursive')
self.excludes = config.get('job_builder', 'exclude').split(os.pathsep)
# The way we want to do things moving forward:
self.jenkins['url'] = config.get(self.section, 'url')
self.jenkins['user'] = self.user
self.jenkins['password'] = self.password
self.jenkins['timeout'] = self.timeout
self.builder['ignore_cache'] = self.ignore_cache
self.builder['flush_cache'] = self.flush_cache
self.builder['plugins_info'] = self.plugins_info
# keep descriptions ? (used by yamlparser)
keep_desc = False
if (config and config.has_section('job_builder')
and config.has_option('job_builder', 'keep_descriptions')):
keep_desc = config.getboolean('job_builder', 'keep_descriptions')
self.yamlparser['keep_descriptions'] = keep_desc
# figure out the include path (used by yamlparser)
path = ["."]
if (config and config.has_section('job_builder')
and config.has_option('job_builder', 'include_path')):
path = config.get('job_builder', 'include_path').split(':')
self.yamlparser['include_path'] = path
# allow duplicates?
allow_duplicates = False
if config and config.has_option('job_builder', 'allow_duplicates'):
allow_duplicates = config.getboolean('job_builder',
'allow_duplicates')
self.yamlparser['allow_duplicates'] = allow_duplicates
# allow empty variables?
self.yamlparser['allow_empty_variables'] = (
self.allow_empty_variables
or config and config.has_section('job_builder')
and config.has_option('job_builder', 'allow_empty_variables')
and config.getboolean('job_builder', 'allow_empty_variables'))
def git(self, xml_parent, data):
logger = logging.getLogger("%s:pipeline-git" % __name__)
scm = XML.SubElement(xml_parent, 'scm',
{'class': 'hudson.plugins.git.GitSCM'})
scm.attrib['plugin'] = 'git'
XML.SubElement(scm, 'configVersion').text = '2'
user = XML.SubElement(scm, 'userRemoteConfigs')
if 'remotes' not in data:
data['remotes'] = [{data.get('name', 'origin'): data.copy()}]
for remoteData in data['remotes']:
huser = XML.SubElement(user, 'hudson.plugins.git.UserRemoteConfig')
remoteName = next(iter(remoteData.keys()))
#XML.SubElement(huser, 'name').text = remoteName
remoteParams = next(iter(remoteData.values()))
if 'url' in remoteParams:
remoteURL = remoteParams['url']
else:
raise JenkinsJobsException(
'Must specify a url for git remote \"' + remoteName + '"')
XML.SubElement(huser, 'url').text = remoteURL
if 'credentials-id' in remoteParams:
credentialsId = remoteParams['credentials-id']
XML.SubElement(huser, 'credentialsId').text = credentialsId
xml_branches = XML.SubElement(scm, 'branches')
branches = data.get('branches', ['**'])
for branch in branches:
bspec = XML.SubElement(xml_branches,
'hudson.plugins.git.BranchSpec')
XML.SubElement(bspec, 'name').text = branch
# add addition elements
#<doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
#<submoduleCfg class="list"/>
mappingOld = [
# option, xml name, default value (text), attributes (hard coded)
(None, 'doGenerateSubmoduleConfigurations', False),
(None, 'submoduleCfg', '', {
'class': 'list'
}),
]
mapping = [
# option, xml name, default value (text), attributes (hard coded)
("ignore-notify", "ignoreNotifyCommit", False),
("shallow-clone", "useShallowClone", False),
]
# first adding the deprecated options
for elem in mappingOld:
(optname, xmlname, val) = elem[:3]
attrs = {}
attrs = {}
if len(elem) >= 4:
attrs = elem[3]
xe = XML.SubElement(scm, xmlname, attrs)
if optname and optname in data:
val = data[optname]
if type(val) == bool:
xe.text = str(val).lower()
else:
xe.text = val
exts_node = XML.SubElement(scm, 'extensions')
for elem in mapping:
(optname, xmlname, val) = elem[:3]
attrs = {}
if len(elem) >= 4:
attrs = elem[3]
xe = XML.SubElement(exts_node, xmlname, attrs)
if optname and optname in data:
val = data[optname]
if type(val) == bool:
xe.text = str(val).lower()
else:
xe.text = val
def git(self, xml_parent, data):
"""yaml: git
Specifies the git SCM repository for this job.
Requires the Jenkins `Git Plugin.
<https://wiki.jenkins-ci.org/display/JENKINS/Git+Plugin>`_
:arg str url: URL of the git repository
:arg str refspec: refspec to fetch
:arg str name: name to fetch
:arg list(str) branches: list of branch specifiers to build
:arg list(str) excluded-users: list of users to ignore revisions from
when polling for changes. (if polling is enabled)
:arg list(str) included-regions: list of file/folders to include
:arg list(str) excluded-regions: list of file/folders to exclude
:arg dict merge:
:merge:
* **remote** (`string`) - name of repo that contains branch to
merge to (default 'origin')
* **branch** (`string`) - name of the branch to merge to
:arg str basedir: location relative to the workspace root to clone to
(default: workspace)
:arg bool skip-tag: Skip tagging
:arg bool shallow-clone: Perform shallow clone
:arg bool prune: Prune remote branches
:arg bool clean: Clean after checkout
:arg bool fastpoll: Use fast remote polling
:arg bool disable-submodules: Disable submodules
:arg bool recursive-submodules: Recursively update submodules
:arg bool use-author: Use author rather than committer in Jenkin's build
changeset
:arg str git-tool: The name of the Git installation to use
:arg str reference-repo: Path of the reference repo to use during clone
:arg str scm-name: The unique scm name for this Git SCM
:arg bool wipe-workspace: Wipe out workspace before build
:arg str browser: what repository browser to use (default '(Auto)')
:arg str browser-url: url for the repository browser
:arg str browser-version: version of the repository browser (GitLab)
:arg str project-name: project name in Gitblit and ViewGit repobrowser
:arg str choosing-strategy: Jenkins class for selecting what to build
:arg str git-config-name: Configure name for Git clone
:arg str git-config-email: Configure email for Git clone
:browser values:
:githubweb:
:fisheye:
:bitbucketweb:
:gitblit:
:gitlab:
:gitoriousweb:
:gitweb:
:redmineweb:
:viewgit:
:choosing-strategy values:
:default:
:inverse:
:gerrit:
Example::
scm:
- git:
url: https://example.com/project.git
branches:
- master
- stable
browser: githubweb
browser-url: http://github.com/foo/example.git
"""
# XXX somebody should write the docs for those with option name =
# None so we have a sensible name/key for it.
mapping = [
# option, xml name, default value (text), attributes (hard coded)
("disable-submodules", 'disableSubmodules', False),
("recursive-submodules", 'recursiveSubmodules', False),
(None, 'doGenerateSubmoduleConfigurations', False),
("use-author", 'authorOrCommitter', False),
("clean", 'clean', False),
("wipe-workspace", 'wipeOutWorkspace', True),
("prune", 'pruneBranches', False),
("fastpoll", 'remotePoll', False),
("git-tool", 'gitTool', "Default"),
(None, 'submoduleCfg', '', {
'class': 'list'
}),
('basedir', 'relativeTargetDir', ''),
('reference-repo', 'reference', ''),
("git-config-name", 'gitConfigName', ''),
("git-config-email", 'gitConfigEmail', ''),
('skip-tag', 'skipTag', False),
('scm-name', 'scmName', ''),
("shallow-clone", "useShallowClone", False),
]
choosing_strategies = {
'default':
'hudson.plugins.git.util.DefaultBuildChooser',
'gerrit': ('com.sonyericsson.hudson.plugins.'
'gerrit.trigger.hudsontrigger.GerritTriggerBuildChooser'),
'inverse':
'hudson.plugins.git.util.InverseBuildChooser',
}
scm = XML.SubElement(xml_parent, 'scm',
{'class': 'hudson.plugins.git.GitSCM'})
XML.SubElement(scm, 'configVersion').text = '2'
user = XML.SubElement(scm, 'userRemoteConfigs')
huser = XML.SubElement(user, 'hudson.plugins.git.UserRemoteConfig')
XML.SubElement(huser, 'name').text = data.get('name', 'origin')
if 'refspec' in data:
refspec = data['refspec']
else:
refspec = '+refs/heads/*:refs/remotes/origin/*'
XML.SubElement(huser, 'refspec').text = refspec
XML.SubElement(huser, 'url').text = data['url']
xml_branches = XML.SubElement(scm, 'branches')
branches = data.get('branches', ['**'])
for branch in branches:
bspec = XML.SubElement(xml_branches, 'hudson.plugins.git.BranchSpec')
XML.SubElement(bspec, 'name').text = branch
excluded_users = '\n'.join(data.get('excluded-users', []))
XML.SubElement(scm, 'excludedUsers').text = excluded_users
if 'included-regions' in data:
include_string = '\n'.join(data['included-regions'])
XML.SubElement(scm, 'includedRegions').text = include_string
if 'excluded-regions' in data:
exclude_string = '\n'.join(data['excluded-regions'])
XML.SubElement(scm, 'excludedRegions').text = exclude_string
if 'merge' in data:
merge = data['merge']
name = merge.get('remote', 'origin')
branch = merge['branch']
urc = XML.SubElement(scm, 'userMergeOptions')
XML.SubElement(urc, 'mergeRemote').text = name
XML.SubElement(urc, 'mergeTarget').text = branch
try:
choosing_strategy = choosing_strategies[data.get(
'choosing-strategy', 'default')]
except KeyError:
raise ValueError('Invalid choosing-strategy %r' %
data.get('choosing-strategy'))
XML.SubElement(scm, 'buildChooser', {'class': choosing_strategy})
for elem in mapping:
(optname, xmlname, val) = elem[:3]
attrs = {}
if len(elem) >= 4:
attrs = elem[3]
xe = XML.SubElement(scm, xmlname, attrs)
if optname and optname in data:
val = data[optname]
if type(val) == bool:
xe.text = str(val).lower()
else:
xe.text = val
browser = data.get('browser', 'auto')
browserdict = {
'githubweb': 'GithubWeb',
'fisheye': 'FisheyeGitRepositoryBrowser',
'bitbucketweb': 'BitbucketWeb',
'cgit': 'CGit',
'gitblit': 'GitBlitRepositoryBrowser',
'gitlab': 'GitLab',
'gitoriousweb': 'GitoriousWeb',
'gitweb': 'GitWeb',
'redmineweb': 'RedmineWeb',
'viewgit': 'ViewGitWeb',
'auto': 'auto'
}
if browser not in browserdict:
raise JenkinsJobsException("Browser entered is not valid must be one "
"of: githubweb, fisheye, bitbucketweb, "
"cgit, gitblit, gitlab, gitoriousweb, "
"gitweb, redmineweb, viewgit, or auto")
if browser != 'auto':
bc = XML.SubElement(
scm, 'browser',
{'class': 'hudson.plugins.git.browser.' + browserdict[browser]})
XML.SubElement(bc, 'url').text = data['browser-url']
if browser in ['gitblit', 'viewgit']:
XML.SubElement(bc, 'projectName').text = str(
data.get('project-name', ''))
if browser == 'gitlab':
XML.SubElement(bc, 'version').text = str(
data.get('browser-version', '0.0'))
def _setup(self):
config = self.config_parser
logger.debug("Config: {0}".format(config))
# check the ignore_cache setting
ignore_cache = False
if config.has_option(self._section, "ignore_cache"):
logger.warning(
"ignore_cache option should be moved to the "
"[job_builder] section in the config file, the "
"one specified in the [jenkins] section will be "
"ignored in the future"
)
ignore_cache = config.getboolean(self._section, "ignore_cache")
elif config.has_option("job_builder", "ignore_cache"):
ignore_cache = config.getboolean("job_builder", "ignore_cache")
self.builder["ignore_cache"] = ignore_cache
# check the flush_cache setting
flush_cache = False
if config.has_option("job_builder", "flush_cache"):
flush_cache = config.getboolean("job_builder", "flush_cache")
self.builder["flush_cache"] = flush_cache
# check the print_job_urls setting
if config.has_option("job_builder", "print_job_urls"):
self.print_job_urls = config.getboolean("job_builder", "print_job_urls")
# Jenkins supports access as an anonymous user, which can be used to
# ensure read-only behaviour when querying the version of plugins
# installed for test mode to generate XML output matching what will be
# uploaded. To enable must pass 'None' as the value for user and
# password to python-jenkins
#
# catching 'TypeError' is a workaround for python 2.6 interpolation
# error
# https://bugs.launchpad.net/openstack-ci/+bug/1259631
try:
user = config.get(self._section, "user")
except (TypeError, configparser.NoOptionError):
user = None
self.jenkins["user"] = user
try:
password = config.get(self._section, "password")
except (TypeError, configparser.NoOptionError):
password = None
self.jenkins["password"] = password
# None -- no timeout, blocking mode; same as setblocking(True)
# 0.0 -- non-blocking mode; same as setblocking(False) <--- default
# > 0 -- timeout mode; operations time out after timeout seconds
# < 0 -- illegal; raises an exception
# to retain the default must use
# "timeout=jenkins_jobs.builder._DEFAULT_TIMEOUT" or not set timeout at
# all.
try:
timeout = config.getfloat(self._section, "timeout")
except (ValueError):
raise JenkinsJobsException("Jenkins timeout config is invalid")
except (TypeError, configparser.NoOptionError):
timeout = builder._DEFAULT_TIMEOUT
self.jenkins["timeout"] = timeout
plugins_info = None
if config.has_option(
self._section, "query_plugins_info"
) and not config.getboolean(self._section, "query_plugins_info"):
logger.debug("Skipping plugin info retrieval")
plugins_info = []
self.builder["plugins_info"] = plugins_info
self.recursive = config.getboolean("job_builder", "recursive")
self.excludes = config.get("job_builder", "exclude").split(os.pathsep)
# The way we want to do things moving forward:
self.jenkins["url"] = config.get(self._section, "url")
self.builder["print_job_urls"] = self.print_job_urls
# keep descriptions ? (used by yamlparser)
keep_desc = False
if (
config
and config.has_section("job_builder")
and config.has_option("job_builder", "keep_descriptions")
):
keep_desc = config.getboolean("job_builder", "keep_descriptions")
self.yamlparser["keep_descriptions"] = keep_desc
# figure out the include path (used by yamlparser)
path = ["."]
if (
config
and config.has_section("job_builder")
and config.has_option("job_builder", "include_path")
):
path = config.get("job_builder", "include_path").split(":")
self.yamlparser["include_path"] = path
# allow duplicates?
allow_duplicates = False
if config and config.has_option("job_builder", "allow_duplicates"):
allow_duplicates = config.getboolean("job_builder", "allow_duplicates")
self.yamlparser["allow_duplicates"] = allow_duplicates
# allow empty variables?
self.yamlparser["allow_empty_variables"] = (
config
and config.has_section("job_builder")
and config.has_option("job_builder", "allow_empty_variables")
and config.getboolean("job_builder", "allow_empty_variables")
)
# retain anchors across files?
retain_anchors = False
if config and config.has_option("job_builder", "retain_anchors"):
retain_anchors = config.getboolean("job_builder", "retain_anchors")
self.yamlparser["retain_anchors"] = retain_anchors
update = None
if (
config
and config.has_section("job_builder")
and config.has_option("job_builder", "update")
):
update = config.get("job_builder", "update")
self.builder["update"] = update
def cvs(parser, xml_parent, data):
"""yaml: cvs
Specifies the CVS SCM repository for this job.
Requires the Jenkins :jenkins-wiki:`CVS Plugin <CVS+Plugin>`.
:arg list repos: List of CVS repositories. (required)
:Repos:
* **root** (`str`) -- The CVS connection string Jenkins uses to
connect to the server. The format is :protocol:user@host:path
(required)
* **locations** (`list`) -- List of locations. (required)
:Locations:
* **type** (`str`) -- Type of location.
:supported values:
* **HEAD** - (default)
* **BRANCH**
* **TAG**
* **name** (`str`) -- Name of location. Only valid in case
of 'BRANCH' or 'TAG' location type. (default '')
* **use-head** (`bool`) -- Use Head if not found. Only
valid in case of 'BRANCH' or 'TAG' location type.
(default false)
* **modules** (`list`) -- List of modules. (required)
:Modules:
* **remote** -- The name of the module in the
repository at CVSROOT. (required)
* **local-name** -- The name to be applied to
this module in the local workspace. If blank,
the remote module name will be used.
(default '')
* **excluded-regions** (`list str`) -- Patterns for excluding
regions. (optional)
* **compression-level** (`int`) -- Compression level. Must be a
number between -1 and 9 inclusive. Choose -1 for System Default.
(default -1)
:arg bool use-update: If true, Jenkins will use 'cvs update' whenever
possible for builds. This makes a build faster. But this also causes the
artifacts from the previous build to remain in the file system when a
new build starts, making it not a true clean build. (default true)
:arg bool prune-empty: Remove empty directories after checkout using the
CVS '-P' option. (default true)
:arg bool skip-changelog: Prevent the changelog being generated after
checkout has completed. (default false)
:arg bool show-all-output: Instructs CVS to show all logging output. CVS
normally runs in quiet mode but this option disables that.
(default false)
:arg bool clean-checkout: Perform clean checkout on failed update.
(default false)
:arg bool clean-copy: Force clean copy for locally modified files.
(default false)
Example
.. literalinclude:: /../../tests/scm/fixtures/cvs001.yaml
:language: yaml
.. literalinclude:: /../../tests/scm/fixtures/cvs002.yaml
:language: yaml
"""
prefix = 'hudson.scm.'
valid_loc_types = {'HEAD': 'Head', 'TAG': 'Tag', 'BRANCH': 'Branch'}
cvs = XML.SubElement(xml_parent, 'scm', {'class': prefix + 'CVSSCM'})
repos = data.get('repos')
if not repos:
raise JenkinsJobsException("'repos' empty or missing")
repos_tag = XML.SubElement(cvs, 'repositories')
for repo in repos:
repo_tag = XML.SubElement(repos_tag, prefix + 'CvsRepository')
try:
XML.SubElement(repo_tag, 'cvsRoot').text = repo['root']
except KeyError:
raise MissingAttributeError('root')
items_tag = XML.SubElement(repo_tag, 'repositoryItems')
locations = repo.get('locations')
if not locations:
raise JenkinsJobsException("'locations' empty or missing")
for location in locations:
item_tag = XML.SubElement(items_tag, prefix + 'CvsRepositoryItem')
loc_type = location.get('type', 'HEAD')
if loc_type not in valid_loc_types:
raise InvalidAttributeError('type', loc_type, valid_loc_types)
loc_class = ('{0}CvsRepositoryLocation${1}Repository'
'Location').format(prefix, valid_loc_types[loc_type])
loc_tag = XML.SubElement(item_tag, 'location',
{'class': loc_class})
XML.SubElement(loc_tag, 'locationType').text = loc_type
if loc_type == 'TAG' or loc_type == 'BRANCH':
XML.SubElement(loc_tag,
'locationName').text = location.get('name', '')
XML.SubElement(loc_tag, 'useHeadIfNotFound').text = str(
location.get('use-head', False)).lower()
modules = location.get('modules')
if not modules:
raise JenkinsJobsException("'modules' empty or missing")
modules_tag = XML.SubElement(item_tag, 'modules')
for module in modules:
module_tag = XML.SubElement(modules_tag, prefix + 'CvsModule')
try:
XML.SubElement(module_tag,
'remoteName').text = module['remote']
except KeyError:
raise MissingAttributeError('remote')
XML.SubElement(module_tag, 'localName').text = module.get(
'local-name', '')
excluded = repo.get('excluded-regions', [])
excluded_tag = XML.SubElement(repo_tag, 'excludedRegions')
for pattern in excluded:
pattern_tag = XML.SubElement(excluded_tag,
prefix + 'ExcludedRegion')
XML.SubElement(pattern_tag, 'pattern').text = pattern
compression_level = repo.get('compression-level', '-1')
if int(compression_level) not in range(-1, 10):
raise InvalidAttributeError('compression-level', compression_level,
range(-1, 10))
XML.SubElement(repo_tag, 'compressionLevel').text = compression_level
mapping = [('use-update', 'canUseUpdate', True),
('prune-empty', 'pruneEmptyDirectories', True),
('skip-changelog', 'skipChangeLog', False),
('show-all-output', 'disableCvsQuiet', False),
('clean-checkout', 'cleanOnFailedUpdate', False),
('clean-copy', 'forceCleanCopy', False)]
for elem in mapping:
opt, xml_tag, val = elem[:]
XML.SubElement(cvs, xml_tag).text = str(data.get(opt, val)).lower()
def main(argv=None):
# We default argv to None and assign to sys.argv[1:] below because having
# an argument default value be a mutable type in Python is a gotcha. See
# http://bit.ly/1o18Vff
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser()
subparser = parser.add_subparsers(help='update, test or delete job',
dest='command')
parser_update = subparser.add_parser('update')
parser_update.add_argument('path', help='path to YAML file or directory')
parser_update.add_argument('names', help='name(s) of job(s)', nargs='*')
parser_update.add_argument(
'--delete-old',
help='delete obsolete jobs',
action='store_true',
dest='delete_old',
default=False,
)
parser_test = subparser.add_parser('test')
parser_test.add_argument('path',
help='path to YAML file or directory',
nargs='?',
default=sys.stdin)
parser_test.add_argument('-o',
dest='output_dir',
default=sys.stdout,
help='path to output XML')
parser_test.add_argument('name', help='name(s) of job(s)', nargs='*')
parser_delete = subparser.add_parser('delete')
parser_delete.add_argument('name', help='name of job', nargs='+')
parser_delete.add_argument('-p',
'--path',
default=None,
help='path to YAML file or directory')
subparser.add_parser('delete-all',
help='delete *ALL* jobs from Jenkins server, '
'including those not managed by Jenkins Job '
'Builder.')
parser.add_argument('--conf', dest='conf', help='configuration file')
parser.add_argument('-l',
'--log_level',
dest='log_level',
default='info',
help="log level (default: %(default)s)")
parser.add_argument(
'--ignore-cache',
action='store_true',
dest='ignore_cache',
default=False,
help='ignore the cache and update the jobs anyhow (that will only '
'flush the specified jobs cache)')
parser.add_argument('--flush-cache',
action='store_true',
dest='flush_cache',
default=False,
help='flush all the cache entries before updating')
options = parser.parse_args(argv)
options.log_level = getattr(logging, options.log_level.upper(),
logging.INFO)
logging.basicConfig(level=options.log_level)
logger = logging.getLogger()
conf = '/etc/jenkins_jobs/jenkins_jobs.ini'
if options.conf:
conf = options.conf
else:
# Fallback to script directory
localconf = os.path.join(os.path.dirname(__file__), 'jenkins_jobs.ini')
if os.path.isfile(localconf):
conf = localconf
config = ConfigParser.ConfigParser()
## Load default config always
config.readfp(cStringIO.StringIO(DEFAULT_CONF))
if options.command == 'test':
logger.debug("Not reading config for test output generation")
elif os.path.isfile(conf):
logger.debug("Reading config from {0}".format(conf))
conffp = open(conf, 'r')
config.readfp(conffp)
else:
raise JenkinsJobsException(
"A valid configuration file is required when not run as a test"
"\n{0} is not a valid .ini file".format(conf))
execute(options, config, logger)
def git(parser, xml_parent, data):
"""yaml: git
Specifies the git SCM repository for this job.
Requires the Jenkins :jenkins-wiki:`Git Plugin <Git+Plugin>`.
:arg str url: URL of the git repository
:arg str credentials-id: ID of credential to use to connect, which is the
last field(a 32-digit hexadecimal code) of the path of URL visible after
you clicked the credential under Jenkins Global credentials. (optional)
:arg str refspec: refspec to fetch (default '+refs/heads/\*:refs/remotes/\
remoteName/\*')
:arg str name: name to fetch (default 'origin')
:arg list(str) remotes: list of remotes to set up (optional, only needed if
multiple remotes need to be set up)
:Remote: * **url** (`string`) - url of remote repo
* **refspec** (`string`) - refspec to fetch (optional)
* **credentials-id** - ID of credential to use to connect,
which is the last field of the path of URL
(a 32-digit hexadecimal code) visible after you clicked
credential under Jenkins Global credentials. (optional)
:arg list(str) branches: list of branch specifiers to build (default '**')
:arg list(str) excluded-users: list of users to ignore revisions from
when polling for changes. (if polling is enabled, optional)
:arg list(str) included-regions: list of file/folders to include (optional)
:arg list(str) excluded-regions: list of file/folders to exclude (optional)
:arg str local-branch: Checkout/merge to local branch (optional)
:arg dict merge:
:merge:
* **remote** (`string`) - name of repo that contains branch to
merge to (default 'origin')
* **branch** (`string`) - name of the branch to merge to
* **strategy** (`string`) - merge strategy. Can be one of
'default', 'resolve', 'recursive', 'octopus', 'ours',
'subtree'. (default 'default')
* **fast-forward-mode** (`string`) - merge fast-forward mode.
Can be one of 'FF', 'FF_ONLY' or 'NO_FF'. (default 'FF')
:arg str basedir: location relative to the workspace root to clone to
(default: workspace)
:arg bool skip-tag: Skip tagging (default false)
:arg bool shallow-clone: Perform shallow clone (default false)
:arg bool prune: Prune remote branches (default false)
:arg bool clean: Clean after checkout (default false)
.. deprecated:: 1.1.1. Please use clean extension format.
:arg bool fastpoll: Use fast remote polling (default false)
:arg bool disable-submodules: Disable submodules (default false)
.. deprecated:: 1.1.1. Please use submodule extension.
:arg bool recursive-submodules: Recursively update submodules (default
false)
.. deprecated:: 1.1.1. Please use submodule extension.
:arg bool use-author: Use author rather than committer in Jenkin's build
changeset (default false)
:arg str git-tool: The name of the Git installation to use (default
'Default')
:arg str reference-repo: Path of the reference repo to use during clone
(optional)
:arg str scm-name: The unique scm name for this Git SCM (optional)
:arg bool ignore-notify: Ignore notifyCommit URL accesses (default false)
:arg str browser: what repository browser to use (default '(Auto)')
:arg str browser-url: url for the repository browser (required if browser
is not '(Auto)', no default)
:arg str browser-version: version of the repository browser (GitLab only,
default '0.0')
:arg str project-name: project name in Gitblit and ViewGit repobrowser
(optional)
:arg str repo-name: repository name in phabricator repobrowser (optional)
:arg str choosing-strategy: Jenkins class for selecting what to build
(default 'default')
:arg str git-config-name: Configure name for Git clone (optional)
:arg str git-config-email: Configure email for Git clone (optional)
:extensions:
:arg dict changelog-against:
:changelog-against:
* **remote** (`string`) - name of repo that contains branch to
create changelog against (default 'origin')
* **branch** (`string`) - name of the branch to create
changelog against (default 'master')
:arg dict clean:
:clean:
* **after** (`bool`) - Clean the workspace after checkout
* **before** (`bool`) - Clean the workspace before checkout
:arg list(str) ignore-commits-with-messages: Revisions committed with
messages matching these patterns will be ignored. (optional)
:arg bool force-polling-using-workspace: Force polling using workspace
(default false)
:arg dict sparse-checkout:
:sparse-checkout:
* **paths** (`list`) - List of paths to sparse checkout.
(optional)
:arg dict submodule:
:submodule:
* **disable** (`bool`) - By disabling support for submodules
you can still keep using basic git plugin functionality
and just have Jenkins to ignore submodules completely as
if they didn't exist.
* **recursive** (`bool`) - Retrieve all submodules recursively
(uses '--recursive' option which requires git>=1.6.5)
* **tracking** (`bool`) - Retrieve the tip of the configured
branch in .gitmodules (Uses '--remote' option which
requires git>=1.8.2)
* **timeout** (`int`) - Specify a timeout (in minutes) for
submodules operations (default: 10).
:arg str timeout: Timeout for git commands in minutes (optional)
:arg bool wipe-workspace: Wipe out workspace before build
(default true)
:browser values:
:auto:
:assemblaweb:
:bitbucketweb:
:cgit:
:fisheye:
:gitblit:
:githubweb:
:gitiles:
:gitlab:
:gitlist:
:gitoriousweb:
:gitweb:
:kiln:
:microsoft-tfs-2013:
:phabricator:
:redmineweb:
:rhodecode:
:stash:
:viewgit:
:choosing-strategy values:
:default:
:inverse:
:gerrit:
Example:
.. literalinclude:: /../../tests/scm/fixtures/git001.yaml
"""
logger = logging.getLogger("%s:git" % __name__)
# XXX somebody should write the docs for those with option name =
# None so we have a sensible name/key for it.
mapping = [
# option, xml name, default value (text), attributes (hard coded)
("disable-submodules", 'disableSubmodules', False),
("recursive-submodules", 'recursiveSubmodules', False),
(None, 'doGenerateSubmoduleConfigurations', False),
("use-author", 'authorOrCommitter', False),
("wipe-workspace", 'wipeOutWorkspace', True),
("prune", 'pruneBranches', False),
("fastpoll", 'remotePoll', False),
("git-tool", 'gitTool', "Default"),
(None, 'submoduleCfg', '', {
'class': 'list'
}),
('basedir', 'relativeTargetDir', ''),
('reference-repo', 'reference', ''),
("git-config-name", 'gitConfigName', ''),
("git-config-email", 'gitConfigEmail', ''),
('skip-tag', 'skipTag', False),
('scm-name', 'scmName', ''),
("shallow-clone", "useShallowClone", False),
("ignore-notify", "ignoreNotifyCommit", False),
]
choosing_strategies = {
'default':
'hudson.plugins.git.util.DefaultBuildChooser',
'gerrit': ('com.sonyericsson.hudson.plugins.'
'gerrit.trigger.hudsontrigger.GerritTriggerBuildChooser'),
'inverse':
'hudson.plugins.git.util.InverseBuildChooser',
}
scm = XML.SubElement(xml_parent, 'scm',
{'class': 'hudson.plugins.git.GitSCM'})
XML.SubElement(scm, 'configVersion').text = '2'
user = XML.SubElement(scm, 'userRemoteConfigs')
if 'remotes' not in data:
data['remotes'] = [{data.get('name', 'origin'): data.copy()}]
for remoteData in data['remotes']:
huser = XML.SubElement(user, 'hudson.plugins.git.UserRemoteConfig')
remoteName = next(iter(remoteData.keys()))
XML.SubElement(huser, 'name').text = remoteName
remoteParams = next(iter(remoteData.values()))
if 'refspec' in remoteParams:
refspec = remoteParams['refspec']
else:
refspec = '+refs/heads/*:refs/remotes/' + remoteName + '/*'
XML.SubElement(huser, 'refspec').text = refspec
if 'url' in remoteParams:
remoteURL = remoteParams['url']
else:
raise JenkinsJobsException('Must specify a url for git remote \"' +
remoteName + '"')
XML.SubElement(huser, 'url').text = remoteURL
if 'credentials-id' in remoteParams:
credentialsId = remoteParams['credentials-id']
XML.SubElement(huser, 'credentialsId').text = credentialsId
xml_branches = XML.SubElement(scm, 'branches')
branches = data.get('branches', ['**'])
for branch in branches:
bspec = XML.SubElement(xml_branches, 'hudson.plugins.git.BranchSpec')
XML.SubElement(bspec, 'name').text = branch
excluded_users = '\n'.join(data.get('excluded-users', []))
XML.SubElement(scm, 'excludedUsers').text = excluded_users
if 'included-regions' in data:
include_string = '\n'.join(data['included-regions'])
XML.SubElement(scm, 'includedRegions').text = include_string
if 'excluded-regions' in data:
exclude_string = '\n'.join(data['excluded-regions'])
XML.SubElement(scm, 'excludedRegions').text = exclude_string
if 'merge' in data:
merge = data['merge']
merge_strategies = [
'default', 'resolve', 'recursive', 'octopus', 'ours', 'subtree'
]
fast_forward_modes = ['FF', 'FF_ONLY', 'NO_FF']
name = merge.get('remote', 'origin')
branch = merge['branch']
urc = XML.SubElement(scm, 'userMergeOptions')
XML.SubElement(urc, 'mergeRemote').text = name
XML.SubElement(urc, 'mergeTarget').text = branch
strategy = merge.get('strategy', 'default')
if strategy not in merge_strategies:
raise InvalidAttributeError('strategy', strategy, merge_strategies)
XML.SubElement(urc, 'mergeStrategy').text = strategy
fast_forward_mode = merge.get('fast-forward-mode', 'FF')
if fast_forward_mode not in fast_forward_modes:
raise InvalidAttributeError('fast-forward-mode', fast_forward_mode,
fast_forward_modes)
XML.SubElement(urc, 'fastForwardMode').text = fast_forward_mode
try:
choosing_strategy = choosing_strategies[data.get(
'choosing-strategy', 'default')]
except KeyError:
raise ValueError('Invalid choosing-strategy %r' %
data.get('choosing-strategy'))
XML.SubElement(scm, 'buildChooser', {'class': choosing_strategy})
for elem in mapping:
(optname, xmlname, val) = elem[:3]
# Throw warning for deprecated settings and skip if the 'submodule' key
# is available.
submodule_cfgs = ['disable-submodules', 'recursive-submodules']
if optname in submodule_cfgs:
if optname in data:
logger.warn("'{0}' is deprecated, please convert to use the "
"'submodule' section instead as support for this "
"top level option will be removed in a future "
"release.".format(optname))
if 'submodule' in data:
continue
attrs = {}
if len(elem) >= 4:
attrs = elem[3]
xe = XML.SubElement(scm, xmlname, attrs)
if optname and optname in data:
val = data[optname]
if type(val) == bool:
xe.text = str(val).lower()
else:
xe.text = val
if 'local-branch' in data:
XML.SubElement(scm, 'localBranch').text = data['local-branch']
exts_node = XML.SubElement(scm, 'extensions')
impl_prefix = 'hudson.plugins.git.extensions.impl.'
if 'changelog-against' in data:
ext_name = impl_prefix + 'ChangelogToBranch'
ext = XML.SubElement(exts_node, ext_name)
opts = XML.SubElement(ext, 'options')
change_remote = data['changelog-against'].get('remote', 'origin')
change_branch = data['changelog-against'].get('branch', 'master')
XML.SubElement(opts, 'compareRemote').text = change_remote
XML.SubElement(opts, 'compareTarget').text = change_branch
if 'clean' in data:
# Keep support for old format 'clean' configuration by checking
# if 'clean' is boolean. Else we're using the new extensions style.
if isinstance(data['clean'], bool):
clean_after = data['clean']
clean_before = False
logger.warn("'clean: bool' configuration format is deprecated, "
"please use the extension style format to configure "
"this option.")
else:
clean_after = data['clean'].get('after', False)
clean_before = data['clean'].get('before', False)
if clean_after:
ext_name = impl_prefix + 'CleanCheckout'
ext = XML.SubElement(exts_node, ext_name)
if clean_before:
ext_name = impl_prefix + 'CleanBeforeCheckout'
ext = XML.SubElement(exts_node, ext_name)
if 'ignore-commits-with-messages' in data:
for msg in data['ignore-commits-with-messages']:
ext_name = impl_prefix + 'MessageExclusion'
ext = XML.SubElement(exts_node, ext_name)
XML.SubElement(ext, 'excludedMessage').text = msg
if 'sparse-checkout' in data:
ext_name = impl_prefix + 'SparseCheckoutPaths'
ext = XML.SubElement(exts_node, ext_name)
sparse_co = XML.SubElement(ext, 'sparseCheckoutPaths')
sparse_paths = data['sparse-checkout'].get('paths')
if sparse_paths is not None:
path_tagname = impl_prefix + 'SparseCheckoutPath'
for path in sparse_paths:
path_tag = XML.SubElement(sparse_co, path_tagname)
XML.SubElement(path_tag, 'path').text = path
if 'submodule' in data:
ext_name = impl_prefix + 'SubmoduleOption'
ext = XML.SubElement(exts_node, ext_name)
XML.SubElement(ext,
'disableSubmodules').text = str(data['submodule'].get(
'disable', False)).lower()
XML.SubElement(ext,
'recursiveSubmodules').text = str(data['submodule'].get(
'recursive', False)).lower()
XML.SubElement(ext,
'trackingSubmodules').text = str(data['submodule'].get(
'tracking', False)).lower()
XML.SubElement(ext, 'timeout').text = str(data['submodule'].get(
'timeout', 10))
if 'timeout' in data:
co = XML.SubElement(exts_node, impl_prefix + 'CheckoutOption')
XML.SubElement(co, 'timeout').text = str(data['timeout'])
polling_using_workspace = str(
data.get('force-polling-using-workspace', False)).lower()
if polling_using_workspace == 'true':
ext_name = impl_prefix + 'DisableRemotePoll'
ext = XML.SubElement(exts_node, ext_name)
# By default we wipe the workspace
wipe_workspace = str(data.get('wipe-workspace', True)).lower()
if wipe_workspace == 'true':
ext_name = impl_prefix + 'WipeWorkspace'
ext = XML.SubElement(exts_node, ext_name)
browser = data.get('browser', 'auto')
browserdict = {
'auto': 'auto',
'assemblaweb': 'AssemblaWeb',
'bitbucketweb': 'BitbucketWeb',
'cgit': 'CGit',
'fisheye': 'FisheyeGitRepositoryBrowser',
'gitblit': 'GitBlitRepositoryBrowser',
'githubweb': 'GithubWeb',
'gitiles': 'Gitiles',
'gitlab': 'GitLab',
'gitlist': 'GitList',
'gitoriousweb': 'GitoriousWeb',
'gitweb': 'GitWeb',
'kiln': 'KilnGit',
'microsoft-tfs-2013': 'TFS2013GitRepositoryBrowser',
'phabricator': 'Phabricator',
'redmineweb': 'RedmineWeb',
'rhodecode': 'RhodeCode',
'stash': 'Stash',
'viewgit': 'ViewGitWeb'
}
if browser not in browserdict:
valid = sorted(browserdict.keys())
raise JenkinsJobsException("Browser entered is not valid must be one "
"of: %s or %s." %
(", ".join(valid[:-1]), valid[-1]))
if browser != 'auto':
bc = XML.SubElement(
scm, 'browser',
{'class': 'hudson.plugins.git.browser.' + browserdict[browser]})
XML.SubElement(bc, 'url').text = data['browser-url']
if browser in ['gitblit', 'viewgit']:
XML.SubElement(bc, 'projectName').text = str(
data.get('project-name', ''))
if browser == 'gitlab':
XML.SubElement(bc, 'version').text = str(
data.get('browser-version', '0.0'))
if browser == 'phabricator':
XML.SubElement(bc, 'repo').text = str(data.get('repo-name', ''))
def expandYaml(self, registry, jobs_glob=None):
changed = True
while changed:
changed = False
for module in registry.modules:
if hasattr(module, 'handle_data'):
if module.handle_data(self.data):
changed = True
self._register_macros()
for default in self.data.get('defaults', {}).values():
self._macro_registry.expand_macros(default)
for job in self.data.get('job', {}).values():
self._macro_registry.expand_macros(job)
if jobs_glob and not matches(job['name'], jobs_glob):
logger.debug("Ignoring job {0}".format(job['name']))
continue
logger.debug("Expanding job '{0}'".format(job['name']))
job = self._applyDefaults(job)
self._formatDescription(job)
self.jobs.append(job)
for view in self.data.get('view', {}).values():
logger.debug("Expanding view '{0}'".format(view['name']))
self._formatDescription(view)
self.views.append(view)
for project in self.data.get('project', {}).values():
logger.debug("Expanding project '{0}'".format(project['name']))
# use a set to check for duplicate job references in projects
seen = set()
for jobspec in project.get('jobs', []):
if isinstance(jobspec, dict):
# Singleton dict containing dict of job-specific params
jobname, jobparams = next(iter(jobspec.items()))
if not isinstance(jobparams, dict):
jobparams = {}
else:
jobname = jobspec
jobparams = {}
job = self._getJob(jobname)
if job:
# Just naming an existing defined job
if jobname in seen:
self._handle_dups("Duplicate job '{0}' specified "
"for project '{1}'"
.format(jobname, project['name']))
seen.add(jobname)
continue
# see if it's a job group
group = self._getJobGroup(jobname)
if group:
for group_jobspec in group['jobs']:
if isinstance(group_jobspec, dict):
group_jobname, group_jobparams = \
next(iter(group_jobspec.items()))
if not isinstance(group_jobparams, dict):
group_jobparams = {}
else:
group_jobname = group_jobspec
group_jobparams = {}
job = self._getJob(group_jobname)
if job:
if group_jobname in seen:
self._handle_dups(
"Duplicate job '{0}' specified for "
"project '{1}'".format(group_jobname,
project['name']))
seen.add(group_jobname)
continue
template = self._getJobTemplate(group_jobname)
# Allow a group to override parameters set by a project
d = type(project)(project)
d.update(jobparams)
d.update(group)
d.update(group_jobparams)
# Except name, since the group's name is not useful
d['name'] = project['name']
if template:
self._expandYamlForTemplateJob(d, template,
jobs_glob)
continue
# see if it's a template
template = self._getJobTemplate(jobname)
if template:
d = type(project)(project)
d.update(jobparams)
self._expandYamlForTemplateJob(d, template, jobs_glob)
else:
raise JenkinsJobsException("Failed to find suitable "
"template named '{0}'"
.format(jobname))
# check for duplicate generated jobs
seen = set()
# walk the list in reverse so that last definition wins
for job in self.jobs[::-1]:
if job['name'] in seen:
self._handle_dups("Duplicate definitions for job '{0}' "
"specified".format(job['name']))
self.jobs.remove(job)
seen.add(job['name'])
return self.jobs, self.views
def hg(self, xml_parent, data):
"""yaml: hg
Specifies the mercurial SCM repository for this job.
Requires the Jenkins :jenkins-wiki:`Mercurial Plugin <Mercurial+Plugin>`.
:arg str url: URL of the hg repository
:arg str credentials-id: ID of credentials to use to connect (optional)
:arg str revision-type: revision type to use (default 'branch')
:arg str revision: the branch or tag name you would like to track
(default 'default')
:arg list(str) modules: reduce unnecessary builds by specifying a list of
"modules" within the repository. A module is a directory name within the
repository that this project lives in. (default '')
:arg bool clean: wipe any local modifications or untracked files in the
repository checkout (default false)
:arg str subdir: check out the Mercurial repository into this
subdirectory of the job's workspace (optional)
:arg bool disable-changelog: do not calculate the Mercurial changelog
for each build (default false)
:arg str browser: what repository browser to use (default 'auto')
:arg str browser-url: url for the repository browser
(required if browser is set)
:browser values:
:fisheye:
:bitbucketweb:
:googlecode:
:hgweb:
:kilnhg:
:rhodecode:
:rhodecodelegacy:
Example:
.. literalinclude:: ../../tests/scm/fixtures/hg02.yaml
"""
scm = XML.SubElement(xml_parent, 'scm',
{'class': 'hudson.plugins.mercurial.MercurialSCM'})
if 'url' in data:
XML.SubElement(scm, 'source').text = data['url']
else:
raise JenkinsJobsException("A top level url must exist")
if 'credentials-id' in data:
XML.SubElement(scm, 'credentialsId').text = data['credentials-id']
revision_type_dict = {
'branch': 'BRANCH',
'tag': 'TAG',
}
try:
revision_type = revision_type_dict[data.get('revision-type', 'branch')]
except KeyError:
raise JenkinsJobsException('Invalid revision-type %r' %
data.get('revision-type'))
XML.SubElement(scm, 'revisionType').text = revision_type
XML.SubElement(scm, 'revision').text = data.get('revision', 'default')
if 'subdir' in data:
XML.SubElement(scm, 'subdir').text = data['subdir']
xc = XML.SubElement(scm, 'clean')
xc.text = str(data.get('clean', False)).lower()
modules = data.get('modules', '')
if isinstance(modules, list):
modules = " ".join(modules)
XML.SubElement(scm, 'modules').text = modules
xd = XML.SubElement(scm, 'disableChangeLog')
xd.text = str(data.get('disable-changelog', False)).lower()
browser = data.get('browser', 'auto')
browserdict = {
'auto': '',
'bitbucket': 'BitBucket',
'fisheye': 'FishEye',
'googlecode': 'GoogleCode',
'hgweb': 'HgWeb',
'kilnhg': 'KilnHG',
'rhodecode': 'RhodeCode',
'rhodecode-pre-1.2.0': 'RhodeCodeLegacy'
}
if browser not in browserdict:
raise JenkinsJobsException("Browser entered is not valid must be one "
"of: %s" % ", ".join(browserdict.keys()))
if browser != 'auto':
bc = XML.SubElement(scm, 'browser', {
'class':
'hudson.plugins.mercurial.browser.' + browserdict[browser]
})
if 'browser-url' in data:
XML.SubElement(bc, 'url').text = data['browser-url']
else:
raise JenkinsJobsException("A browser-url must be specified along "
"with browser.")
def extended_choice_param(parser, xml_parent, data):
"""yaml: extended-choice
Creates an extended choice parameter where values can be read from a file
Requires the Jenkins :jenkins-wiki:`Extended Choice Parameter Plugin
<Extended+Choice+Parameter+plugin>`.
:arg str name: name of the parameter
:arg str description: description of the parameter
(optional, default '')
:arg str property-file: location of property file to read from
(optional, default '')
:arg str property-key: key for the property-file (optional, default '')
:arg bool quote-value: whether to put quotes around the property
when passing to Jenkins (optional, default false)
:arg str visible-items: number of items to show in the list
(optional, default 5)
:arg str type: type of select, can be single-select, multi-select,
radio, checkbox or textbox (optional, default single-select)
:arg str value: comma separated list of values for the single select
or multi-select box (optional, default '')
:arg str default-value: used to set the initial selection of the
single-select or multi-select box (optional, default '')
:arg str default-property-file: location of property file when default
value needs to come from a property file (optional, default '')
:arg str default-property-key: key for the default property file
(optional, default '')
:arg str multi-select-delimiter: value between selections when the
parameter is a multi-select (optiona, default ',')
Example:
.. literalinclude:: \
/../../tests/parameters/fixtures/extended-choice-param001.yaml
:language: yaml
"""
pdef = base_param(parser, xml_parent, data, False,
'com.cwctravel.hudson.plugins.'
'extended__choice__parameter.'
'ExtendedChoiceParameterDefinition')
XML.SubElement(pdef, 'value').text = data.get('value', '')
XML.SubElement(pdef, 'visibleItemCount').text = str(data.get(
'visible-items', data.get('visible-item-count', 5)))
XML.SubElement(pdef, 'multiSelectDelimiter').text = data.get(
'multi-select-delimiter', ',')
XML.SubElement(pdef, 'quoteValue').text = str(data.get('quote-value',
False)).lower()
XML.SubElement(pdef, 'defaultValue').text = data.get(
'default-value', '')
choice = data.get('type', 'single-select')
choicedict = {'single-select': 'PT_SINGLE_SELECT',
'multi-select': 'PT_MULTI_SELECT',
'radio': 'PT_RADIO',
'checkbox': 'PT_CHECKBOX',
'textbox': 'PT_TEXTBOX',
'PT_SINGLE_SELECT': 'PT_SINGLE_SELECT',
'PT_MULTI_SELECT': 'PT_MULTI_SELECT',
'PT_RADIO': 'PT_RADIO',
'PT_CHECKBOX': 'PT_CHECKBOX',
'PT_TEXTBOX': 'PT_TEXTBOX'}
if choice in choicedict:
XML.SubElement(pdef, 'type').text = choicedict[choice]
else:
raise JenkinsJobsException("Type entered is not valid, must be one "
"of: single-select, multi-select, radio, "
"textbox or checkbox")
XML.SubElement(pdef, 'propertyFile').text = data.get('property-file', '')
XML.SubElement(pdef, 'propertyKey').text = data.get('property-key', '')
XML.SubElement(pdef, 'defaultPropertyFile').text = data.get(
'default-property-file', '')
XML.SubElement(pdef, 'defaultPropertyKey').text = data.get(
'default-property-key', '')
def repo(parser, xml_parent, data):
"""yaml: repo
Specifies the repo SCM repository for this job.
Requires the Jenkins `Repo Plugin.
<https://wiki.jenkins-ci.org/display/JENKINS/Repo+Plugin>`_
:arg str manifest-url: URL of the repo manifest
:arg str manifest-branch: The branch of the manifest to use (optional)
:arg str manifest-file: Initial manifest file to use when initialising
(optional)
:arg str manifest-group: Only retrieve those projects in the manifest
tagged with the provided group name (optional)
:arg str destination-dir: Location relative to the workspace root to clone
under (optional)
:arg str repo-url: custom url to retrieve the repo application (optional)
:arg str mirror-dir: Path to mirror directory to reference when
initialising (optional)
:arg int jobs: Number of projects to fetch simultaneously (default 0)
:arg bool current-branch: Fetch only the current branch from the server
(default true)
:arg bool quiet: Make repo more quiet
(default true)
:arg str local-manifest: Contents of .repo/local_manifest.xml, written
prior to calling sync (optional)
Example:
.. literalinclude:: /../../tests/scm/fixtures/repo001.yaml
"""
scm = XML.SubElement(xml_parent, 'scm',
{'class': 'hudson.plugins.repo.RepoScm'})
if 'manifest-url' in data:
XML.SubElement(scm, 'manifestRepositoryUrl').text = \
data['manifest-url']
else:
raise JenkinsJobsException("Must specify a manifest url")
mapping = [
# option, xml name, default value
("manifest-branch", 'manifestBranch', ''),
("manifest-file", 'manifestFile', ''),
("manifest-group", 'manifestGroup', ''),
("destination-dir", 'destinationDir', ''),
("repo-url", 'repoUrl', ''),
("mirror-dir", 'mirrorDir', ''),
("jobs", 'jobs', 0),
("current-branch", 'currentBranch', True),
("quiet", 'quiet', True),
("local-manifest", 'localManifest', ''),
]
for elem in mapping:
(optname, xmlname, val) = elem
val = data.get(optname, val)
# Skip adding xml entry if default is empty string and no value given
if not val and elem[2] is '':
continue
xe = XML.SubElement(scm, xmlname)
if type(elem[2]) == bool:
xe.text = str(val).lower()
else:
xe.text = str(val)
def dispatch(self,
component_type,
parser,
xml_parent,
component,
template_data={}):
"""This is a method that you can call from your implementation of
Base.gen_xml or component. It allows modules to define a type
of component, and benefit from extensibility via Python
entry points and Jenkins Job Builder :ref:`Macros <macro>`.
:arg string component_type: the name of the component
(e.g., `builder`)
:arg YAMLParser parser: the global YMAL Parser
:arg Element xml_parent: the parent XML element
:arg dict template_data: values that should be interpolated into
the component definition
See :py:class:`jenkins_jobs.modules.base.Base` for how to register
components of a module.
See the Publishers module for a simple example of how to use
this method.
"""
if component_type not in self.modules_by_component_type:
raise JenkinsJobsException("Unknown component type: "
"'{0}'.".format(component_type))
component_list_type = self.modules_by_component_type[component_type] \
.component_list_type
if isinstance(component, dict):
# The component is a sigleton dictionary of name: dict(args)
name, component_data = component.items()[0]
if template_data:
# Template data contains values that should be interpolated
# into the component definition
s = yaml.dump(component_data, default_flow_style=False)
s = s.format(**template_data)
component_data = yaml.load(s)
else:
# The component is a simple string name, eg "run-tests"
name = component
component_data = {}
# Look for a component function defined in an entry point
cache_key = '%s:%s' % (component_list_type, name)
eps = ModuleRegistry.entry_points_cache.get(cache_key)
if eps is None:
eps = list(
pkg_resources.iter_entry_points(
group='jenkins_jobs.{0}'.format(component_list_type),
name=name))
if len(eps) > 1:
raise JenkinsJobsException(
"Duplicate entry point found for component type: '{0}',"
"name: '{1}'".format(component_type, name))
elif len(eps) == 1:
ModuleRegistry.entry_points_cache[cache_key] = eps
logger.debug("Cached entry point %s = %s", cache_key,
ModuleRegistry.entry_points_cache[cache_key])
if len(eps) == 1:
func = eps[0].load()
func(parser, xml_parent, component_data)
else:
# Otherwise, see if it's defined as a macro
component = parser.data.get(component_type, {}).get(name)
if component:
for b in component[component_list_type]:
# Pass component_data in as template data to this function
# so that if the macro is invoked with arguments,
# the arguments are interpolated into the real defn.
self.dispatch(component_type, parser, xml_parent, b,
component_data)
else:
raise JenkinsJobsException(
"Unknown entry point or macro '{0}'"
" for component type: '{1}'.".format(name, component_type))
def shining_panda(parser, xml_parent, data):
"""yaml: shining-panda
Execute a command inside various python environments. Requires the Jenkins
`ShiningPanda plugin
<https://wiki.jenkins-ci.org/display/JENKINS/ShiningPanda+Plugin>`_.
:arg str build-environment: Building environment to set up (Required).
:build-environment values:
* **python**: Use a python installation configured in Jenkins.
* **custom**: Use a manually installed python.
* **virtualenv**: Create a virtualenv
For the **python** environment
:arg str python-version: Name of the python installation to use.
Must match one of the configured installations on server \
configuration
(default: System-CPython-2.7)
For the **custom** environment:
:arg str home: path to the home folder of the custom installation \
(Required)
For the **virtualenv** environment:
:arg str python-version: Name of the python installation to use.
Must match one of the configured installations on server \
configuration
(default: System-CPython-2.7)
:arg str name: Name of this virtualenv. Two virtualenv builders with \
the same name will use the same virtualenv installation (optional)
:arg bool clear: If true, delete and recreate virtualenv on each build.
(default: false)
:arg bool use-distribute: if true use distribute, if false use \
setuptools. (default: true)
:arg bool system-site-packages: if true, give access to the global
site-packages directory to the virtualenv. (default: false)
Common to all environments:
:arg str nature: Nature of the command field. (default: shell)
:nature values:
* **shell**: execute the Command contents with default shell
* **xshell**: like **shell** but performs platform conversion \
first
* **python**: execute the Command contents with the Python \
executable
:arg str command: The command to execute
:arg bool ignore-exit-code: mark the build as failure if any of the
commands exits with a non-zero exit code. (default: false)
Examples:
.. literalinclude:: \
/../../tests/builders/fixtures/shining-panda-pythonenv.yaml
:language: yaml
.. literalinclude:: \
/../../tests/builders/fixtures/shining-panda-customenv.yaml
:language: yaml
.. literalinclude:: \
/../../tests/builders/fixtures/shining-panda-virtualenv.yaml
:language: yaml
"""
pluginelementpart = 'jenkins.plugins.shiningpanda.builders.'
buildenvdict = {
'custom': 'CustomPythonBuilder',
'virtualenv': 'VirtualenvBuilder',
'python': 'PythonBuilder'
}
envs = (buildenvdict.keys())
try:
buildenv = data['build-environment']
except KeyError:
raise JenkinsJobsException("A build-environment is required")
if buildenv not in envs:
errorstring = (
"build-environment '%s' is invalid. Must be one of %s." %
(buildenv, ', '.join("'{0}'".format(env) for env in envs)))
raise JenkinsJobsException(errorstring)
t = XML.SubElement(xml_parent,
'%s%s' % (pluginelementpart, buildenvdict[buildenv]))
if buildenv in ('python', 'virtualenv'):
XML.SubElement(t,
'pythonName').text = data.get("python-version",
"System-CPython-2.7")
if buildenv in ('custom'):
try:
homevalue = data["home"]
except KeyError:
raise JenkinsJobsException("'home' argument is required for the"
" 'custom' environment")
XML.SubElement(t, 'home').text = homevalue
if buildenv in ('virtualenv'):
XML.SubElement(t, 'home').text = data.get("name", "")
clear = data.get("clear", False)
XML.SubElement(t, 'clear').text = str(clear).lower()
use_distribute = data.get('use-distribute', False)
XML.SubElement(t, 'useDistribute').text = str(use_distribute).lower()
system_site_packages = data.get('system-site-packages', False)
XML.SubElement(
t, 'systemSitePackages').text = str(system_site_packages).lower()
# Common arguments
nature = data.get('nature', 'shell')
naturetuple = ('shell', 'xshell', 'python')
if nature not in naturetuple:
errorstring = ("nature '%s' is not valid: must be one of %s." %
(nature, ', '.join("'{0}'".format(naturevalue)
for naturevalue in naturetuple)))
raise JenkinsJobsException(errorstring)
XML.SubElement(t, 'nature').text = nature
XML.SubElement(t, 'command').text = data.get("command", "")
ignore_exit_code = data.get('ignore-exit-code', False)
XML.SubElement(t, 'ignoreExitCode').text = str(ignore_exit_code).lower()
