def test_yaml_snippet(self):
config = self._get_config()
expected_xml = self._read_utf8_content()
parser = YamlParser(config)
parser.parse(self.in_filename)
registry = ModuleRegistry(config)
registry.set_parser_data(parser.data)
job_data_list, view_data_list = parser.expandYaml(registry)
# Generate the XML tree
xml_generator = XmlJobGenerator(registry)
xml_jobs = xml_generator.generateXML(job_data_list)
xml_jobs.sort(key=AlphanumSort)
# Prettify generated XML
pretty_xml = u"\n".join(job.output().decode('utf-8')
for job in xml_jobs)
self.assertThat(
pretty_xml,
testtools.matchers.DocTestMatches(
expected_xml, doctest.ELLIPSIS | doctest.REPORT_NDIFF))
def _generate_xmljobs(self, options, jjb_config=None):
builder = Builder(jjb_config)
logger.info("Updating jobs in {0} ({1})".format(
options.path, options.names))
orig = time.time()
# Generate XML
parser = YamlParser(jjb_config)
registry = ModuleRegistry(jjb_config, builder.plugins_list)
xml_generator = XmlJobGenerator(registry)
parser.load_files(options.path)
registry.set_parser_data(parser.data)
job_data_list = parser.expandYaml(registry, options.names)
xml_jobs = xml_generator.generateXML(job_data_list)
jobs = parser.jobs
step = time.time()
logging.debug('%d XML files generated in %ss',
len(jobs), str(step - orig))
return builder, xml_jobs
def generate_jjb_xml(self):
"""render jjb yaml to xml"""
jjb_config = self.get_jjb_config()
options_names = [] # normally a list of jobs globs for targeting
files_path = glob.glob("./**/", recursive=True)
parser = YamlParser(jjb_config)
registry = ModuleRegistry(jjb_config, self.plugins_list)
xml_job_generator = XmlJobGeneratorWithRaw(registry)
xml_view_generator = XmlViewGenerator(registry)
parser.load_files(files_path)
registry.set_parser_data(parser.data)
job_data_list, view_data_list = parser.expandYaml(
registry, options_names)
def job_data_filter_wrapper(job_data):
return self._jobs_filter_func(job_data["name"])
xml_jobs = xml_job_generator.generateXML(
filter(job_data_filter_wrapper, job_data_list))
jobs = self.jobs
for xml_job in xml_jobs:
formatted_xml_str = self.xml_dump(xml_job.xml)
jobs[xml_job.name].after_xml = formatted_xml_str
xml_views = xml_view_generator.generateXML(
filter(job_data_filter_wrapper, view_data_list))
views = self.views
for xml_view in xml_views:
views[xml_view.name].after_xml = self.xml_dump(xml_view.xml)
def execute(self, options, jjb_config):
builder = JenkinsManager(jjb_config)
if options.del_jobs and options.del_views:
raise JenkinsJobsException(
'"--views-only" and "--jobs-only" cannot be used together.')
fn = options.path
registry = ModuleRegistry(jjb_config, builder.plugins_list)
parser = YamlParser(jjb_config)
if fn:
parser.load_files(fn)
parser.expandYaml(registry, options.name)
jobs = [j["name"] for j in parser.jobs]
views = [v["name"] for v in parser.views]
else:
jobs = options.name
views = options.name
if options.del_jobs:
builder.delete_jobs(jobs)
elif options.del_views:
builder.delete_views(views)
else:
builder.delete_jobs(jobs)
builder.delete_views(views)
def test_yaml_snippet(self):
if not self.in_filename:
return
jjb_config = self._get_config()
expected_xml = self._read_utf8_content()
yaml_content = self._read_yaml_content(self.in_filename)
plugins_info = None
if self.plugins_info_filename is not None:
plugins_info = self._read_yaml_content(self.plugins_info_filename)
self.addDetail("plugins-info-filename",
text_content(self.plugins_info_filename))
self.addDetail("plugins-info", text_content(str(plugins_info)))
parser = YamlParser(jjb_config)
registry = ModuleRegistry(jjb_config, plugins_info)
registry.set_parser_data(parser.data)
pub = self.klass(registry)
project = None
if ('project-type' in yaml_content):
if (yaml_content['project-type'] == "maven"):
project = project_maven.Maven(registry)
elif (yaml_content['project-type'] == "matrix"):
project = project_matrix.Matrix(registry)
elif (yaml_content['project-type'] == "flow"):
project = project_flow.Flow(registry)
elif (yaml_content['project-type'] == "multijob"):
project = project_multijob.MultiJob(registry)
elif (yaml_content['project-type'] == "externaljob"):
project = project_externaljob.ExternalJob(registry)
if 'view-type' in yaml_content:
if yaml_content['view-type'] == "list":
project = view_list.List(None)
elif yaml_content['view-type'] == "pipeline":
project = view_pipeline.Pipeline(None)
else:
raise InvalidAttributeError('view-type',
yaml_content['view-type'])
if project:
xml_project = project.root_xml(yaml_content)
else:
xml_project = XML.Element('project')
# Generate the XML tree directly with modules/general
pub.gen_xml(xml_project, yaml_content)
# Prettify generated XML
pretty_xml = XmlJob(xml_project, 'fixturejob').output().decode('utf-8')
self.assertThat(
pretty_xml,
testtools.matchers.DocTestMatches(
expected_xml, doctest.ELLIPSIS | doctest.REPORT_NDIFF))
def test_yaml_snippet(self):
config = self._get_config()
expected_xml = (self._read_utf8_content().strip().replace(
"<BLANKLINE>", "").replace("\n\n", "\n"))
parser = YamlParser(config)
parser.parse(self.in_filename)
plugins_info = None
if self.plugins_info_filename:
plugins_info = self._read_yaml_content(self.plugins_info_filename)
self.addDetail("plugins-info-filename",
text_content(self.plugins_info_filename))
self.addDetail("plugins-info", text_content(str(plugins_info)))
registry = ModuleRegistry(config, plugins_info)
registry.set_parser_data(parser.data)
job_data_list, view_data_list = parser.expandYaml(registry)
# Generate the XML tree
xml_generator = XmlJobGenerator(registry)
xml_jobs = xml_generator.generateXML(job_data_list)
xml_jobs.sort(key=AlphanumSort)
# check reference files are under correct path for folders
prefix = os.path.dirname(self.in_filename)
# split using '/' since fullname uses URL path separator
expected_folders = list(
set([
os.path.normpath(
os.path.join(prefix,
"/".join(job_data["name"].split("/")[:-1])))
for job_data in job_data_list
]))
actual_folders = [os.path.dirname(f) for f in self.out_filenames]
six.assertCountEqual(
self,
expected_folders,
actual_folders,
"Output file under wrong path, was '%s', should be '%s'" % (
self.out_filenames[0],
os.path.join(expected_folders[0],
os.path.basename(self.out_filenames[0])),
),
)
# Prettify generated XML
pretty_xml = (u"\n".join(job.output().decode("utf-8")
for job in xml_jobs).strip().replace(
"\n\n", "\n"))
self.assertThat(
pretty_xml,
testtools.matchers.DocTestMatches(
expected_xml, doctest.ELLIPSIS | doctest.REPORT_NDIFF),
)
def __init__(self, jjb_config=None, plugins_info=None):
self.data = {}
self.jobs = []
self.xml_jobs = []
self.jjb_config = jjb_config
self.keep_desc = jjb_config.yamlparser['keep_descriptions']
self.path = jjb_config.yamlparser['include_path']
self.registry = ModuleRegistry(jjb_config, plugins_info)
def get_jobs(self, wd, name):
ini_path = "{}/{}.ini".format(wd, name)
config_path = "{}/config.yaml".format(wd)
args = ["--conf", ini_path, "test", config_path]
jjb = self.get_jjb(args)
builder = JenkinsManager(jjb.jjb_config)
registry = ModuleRegistry(jjb.jjb_config, builder.plugins_list)
parser = YamlParser(jjb.jjb_config)
parser.load_files(jjb.options.path)
jobs, _ = parser.expandYaml(registry, jjb.options.names)
return jobs
def __init__(self, config=None, plugins_info=None):
self.data = {}
self.jobs = []
self.xml_jobs = []
self.config = config
self.registry = ModuleRegistry(self.config, plugins_info)
self.path = ["."]
if self.config:
if config.has_section('job_builder') and \
config.has_option('job_builder', 'include_path'):
self.path = config.get('job_builder',
'include_path').split(':')
self.keep_desc = self.get_keep_desc()
def execute(self, options, jjb_config):
builder = JenkinsManager(jjb_config)
fn = options.path
registry = ModuleRegistry(jjb_config, builder.plugins_list)
parser = YamlParser(jjb_config)
if fn:
parser.load_files(fn)
parser.expandYaml(registry, options.name)
jobs = [j['name'] for j in parser.jobs]
else:
jobs = options.name
builder.delete_jobs(jobs)
def setUp(self):
super(ModuleRegistryPluginInfoTestsWithScenarios, self).setUp()
config = configparser.ConfigParser()
config.readfp(StringIO(cmd.DEFAULT_CONF))
plugin_info = [{'shortName': "HerpDerpPlugin",
'longName': "Blah Blah Blah Plugin"
}]
plugin_info.append({'shortName': "JankyPlugin1",
'longName': "Not A Real Plugin",
'version': self.v1
})
self.addDetail("plugin_info", text_content(str(plugin_info)))
self.registry = ModuleRegistry(config, plugin_info)
def execute(self, options, jjb_config):
builder = Builder(jjb_config)
fn = options.path
registry = ModuleRegistry(jjb_config, builder.plugins_list)
for jobs_glob in options.name:
parser = YamlParser(jjb_config)
if fn:
parser.load_files(fn)
parser.expandYaml(registry, [jobs_glob])
jobs = [j['name'] for j in parser.jobs]
else:
jobs = [jobs_glob]
builder.delete_job(jobs)
def test_yaml_snippet(self):
config = self._get_config()
expected_xml = self._read_utf8_content()
parser = YamlParser(config)
parser.parse(self.in_filename)
registry = ModuleRegistry(config)
registry.set_parser_data(parser.data)
job_data_list, view_data_list = parser.expandYaml(registry)
# Generate the XML tree
xml_generator = XmlJobGenerator(registry)
xml_jobs = xml_generator.generateXML(job_data_list)
xml_jobs.sort(key=AlphanumSort)
# check reference files are under correct path for folders
prefix = os.path.dirname(self.in_filename)
# split using '/' since fullname uses URL path separator
expected_folders = list(
set([
os.path.normpath(
os.path.join(prefix,
'/'.join(job_data['name'].split('/')[:-1])))
for job_data in job_data_list
]))
actual_folders = [os.path.dirname(f) for f in self.out_filenames]
six.assertCountEqual(
self, expected_folders, actual_folders,
"Output file under wrong path, was '%s', should be '%s'" %
(self.out_filenames[0],
os.path.join(expected_folders[0],
os.path.basename(self.out_filenames[0]))))
# Prettify generated XML
pretty_xml = u"\n".join(job.output().decode('utf-8')
for job in xml_jobs)
self.assertThat(
pretty_xml,
testtools.matchers.DocTestMatches(
expected_xml, doctest.ELLIPSIS | doctest.REPORT_NDIFF))
def setUp(self):
super(ModuleRegistryPluginInfoTestsWithScenarios, self).setUp()
jjb_config = JJBConfig()
jjb_config.validate()
plugin_info = [{
"shortName": "HerpDerpPlugin",
"longName": "Blah Blah Blah Plugin"
}]
plugin_info.append({
"shortName": "JankyPlugin1",
"longName": "Not A Real Plugin",
"version": self.v1,
})
self.addDetail("plugin_info", text_content(str(plugin_info)))
self.registry = ModuleRegistry(jjb_config, plugin_info)
def test_retain_anchors_enabled_j2_yaml(self):
"""
Verify that anchors are retained across files and are properly retained when using !j2-yaml.
"""
files = [
"custom_retain_anchors_j2_yaml_include001.yaml",
"custom_retain_anchors_j2_yaml.yaml",
]
jjb_config = JJBConfig()
jjb_config.yamlparser["retain_anchors"] = True
jjb_config.validate()
j = YamlParser(jjb_config)
j.load_files([os.path.join(self.fixtures_path, f) for f in files])
registry = ModuleRegistry(jjb_config, None)
jobs, _ = j.expandYaml(registry)
self.assertEqual(jobs[0]["builders"][0]["shell"],
"docker run ubuntu:latest")
def assert_case(case_name):
case_source, case_result = (os.path.join(BASE_PATH, case_name + ext)
for ext in ['.yml', '.xml'])
jjb_config = JJBConfig()
builder = Builder(jjb_config)
# Generate XML
parser = YamlParser(jjb_config)
registry = ModuleRegistry(jjb_config, builder.plugins_list)
xml_generator = XmlJobGenerator(registry)
parser.load_files(case_source)
registry.set_parser_data(parser.data)
job_data_list = parser.expandYaml(registry, [])
xml_jobs = xml_generator.generateXML(job_data_list)
result_xml = ET.XML(xml_jobs[0].output())
expected_xml = ET.XML(open(case_result).read())
assert ET.tostring(result_xml) == ET.tostring(expected_xml)
def get_job_webhooks_data(self):
job_webhooks_data = {}
for name, wd in self.working_dirs.items():
ini_path = '{}/{}.ini'.format(wd, name)
config_path = '{}/config.yaml'.format(wd)
args = ['--conf', ini_path, 'test', config_path]
jjb = self.get_jjb(args)
builder = JenkinsManager(jjb.jjb_config)
registry = ModuleRegistry(jjb.jjb_config, builder.plugins_list)
parser = YamlParser(jjb.jjb_config)
parser.load_files(jjb.options.path)
jobs, _ = parser.expandYaml(registry, jjb.options.names)
for job in jobs:
try:
project_url_raw = job['properties'][0]['github']['url']
if 'https://github.com' in project_url_raw:
continue
job_url = \
'{}/project/{}'.format(self.instances[name],
job['name'])
project_url = \
project_url_raw.strip('/').replace('.git', '')
gitlab_triggers = job['triggers'][0]['gitlab']
mr_trigger = gitlab_triggers['trigger-merge-request']
trigger = 'mr' if mr_trigger else 'push'
hook = {
'job_url': job_url,
'trigger': trigger,
}
job_webhooks_data.setdefault(project_url, [])
job_webhooks_data[project_url].append(hook)
except KeyError:
continue
return job_webhooks_data
def test_yaml_snippet(self):
if not self.in_filename:
return
jjb_config = self._get_config()
expected_xml = self._read_utf8_content()
yaml_content = self._read_yaml_content(self.in_filename)
plugins_info = None
if self.plugins_info_filename:
plugins_info = self._read_yaml_content(self.plugins_info_filename)
self.addDetail("plugins-info-filename",
text_content(self.plugins_info_filename))
self.addDetail("plugins-info", text_content(str(plugins_info)))
parser = YamlParser(jjb_config)
registry = ModuleRegistry(jjb_config, plugins_info)
registry.set_parser_data(parser.data)
pub = self.klass(registry)
project = None
if ('project-type' in yaml_content):
if (yaml_content['project-type'] == "maven"):
project = project_maven.Maven(registry)
elif (yaml_content['project-type'] == "matrix"):
project = project_matrix.Matrix(registry)
elif (yaml_content['project-type'] == "flow"):
project = project_flow.Flow(registry)
elif (yaml_content['project-type'] == "multijob"):
project = project_multijob.MultiJob(registry)
elif (yaml_content['project-type'] == "multibranch"):
project = project_multibranch.WorkflowMultiBranch(registry)
elif (yaml_content['project-type'] == "multibranch-defaults"):
project = project_multibranch.WorkflowMultiBranchDefaults(
registry) # noqa
elif (yaml_content['project-type'] == "externaljob"):
project = project_externaljob.ExternalJob(registry)
if 'view-type' in yaml_content:
if yaml_content['view-type'] == "list":
project = view_list.List(None)
elif yaml_content['view-type'] == "pipeline":
project = view_pipeline.Pipeline(None)
else:
raise InvalidAttributeError('view-type',
yaml_content['view-type'])
if project:
xml_project = project.root_xml(yaml_content)
else:
xml_project = XML.Element('project')
# Generate the XML tree directly with modules/general
pub.gen_xml(xml_project, yaml_content)
# check output file is under correct path
if 'name' in yaml_content:
prefix = os.path.dirname(self.in_filename)
# split using '/' since fullname uses URL path separator
expected_folders = [
os.path.normpath(
os.path.join(
prefix, '/'.join(
parser._getfullname(yaml_content).split('/')
[:-1])))
]
actual_folders = [os.path.dirname(f) for f in self.out_filenames]
self.assertEquals(
expected_folders, actual_folders,
"Output file under wrong path, was '%s', should be '%s'" %
(self.out_filenames[0],
os.path.join(expected_folders[0],
os.path.basename(self.out_filenames[0]))))
# Prettify generated XML
pretty_xml = XmlJob(xml_project, 'fixturejob').output().decode('utf-8')
self.assertThat(
pretty_xml,
testtools.matchers.DocTestMatches(
expected_xml, doctest.ELLIPSIS | doctest.REPORT_NDIFF))
def test_yaml_snippet(self, mock):
if not self.in_filename:
return
jjb_config = self._get_config()
expected_xml = self._read_utf8_content()
yaml_content = self._read_yaml_content(self.in_filename)
plugins_info = None
if self.plugins_info_filename:
plugins_info = self._read_yaml_content(self.plugins_info_filename)
self.addDetail("plugins-info-filename",
text_content(self.plugins_info_filename))
self.addDetail("plugins-info", text_content(str(plugins_info)))
parser = YamlParser(jjb_config)
e = pkg_resources.EntryPoint.parse
d = pkg_resources.Distribution()
config = configparser.ConfigParser()
config.read(os.path.dirname(__file__) + "/../setup.cfg")
groups = {}
for key in config["entry_points"]:
groups[key] = list()
for line in config["entry_points"][key].split("\n"):
if "" == line.strip():
continue
groups[key].append(e(line, dist=d))
def mock_iter_entry_points(group, name=None):
return (entry for entry in groups[group]
if name is None or name == entry.name)
mock.side_effect = mock_iter_entry_points
registry = ModuleRegistry(jjb_config, plugins_info)
registry.set_parser_data(parser.data)
pub = self.klass(registry)
project = None
if "project-type" in yaml_content:
if yaml_content["project-type"] == "maven":
project = project_maven.Maven(registry)
elif yaml_content["project-type"] == "matrix":
project = project_matrix.Matrix(registry)
elif yaml_content["project-type"] == "flow":
project = project_flow.Flow(registry)
elif yaml_content["project-type"] == "githuborg":
project = project_githuborg.GithubOrganization(registry)
elif yaml_content["project-type"] == "multijob":
project = project_multijob.MultiJob(registry)
elif yaml_content["project-type"] == "multibranch":
project = project_multibranch.WorkflowMultiBranch(registry)
elif yaml_content["project-type"] == "multibranch-defaults":
project = project_multibranch.WorkflowMultiBranchDefaults(
registry) # noqa
elif yaml_content["project-type"] == "externaljob":
project = project_externaljob.ExternalJob(registry)
if "view-type" in yaml_content:
if yaml_content["view-type"] == "all":
project = view_all.All(registry)
elif yaml_content["view-type"] == "delivery_pipeline":
project = view_delivery_pipeline.DeliveryPipeline(registry)
elif yaml_content["view-type"] == "list":
project = view_list.List(registry)
elif yaml_content["view-type"] == "nested":
project = view_nested.Nested(registry)
elif yaml_content["view-type"] == "pipeline":
project = view_pipeline.Pipeline(registry)
elif yaml_content["view-type"] == "sectioned":
project = view_sectioned.Sectioned(registry)
else:
raise InvalidAttributeError("view-type",
yaml_content["view-type"])
if project:
xml_project = project.root_xml(yaml_content)
else:
xml_project = XML.Element("project")
# Generate the XML tree directly with modules/general
pub.gen_xml(xml_project, yaml_content)
# check output file is under correct path
if "name" in yaml_content:
prefix = os.path.dirname(self.in_filename)
# split using '/' since fullname uses URL path separator
expected_folders = [
os.path.normpath(
os.path.join(
prefix,
"/".join(
parser._getfullname(yaml_content).split("/")[:-1]),
))
]
actual_folders = [os.path.dirname(f) for f in self.out_filenames]
self.assertEquals(
expected_folders,
actual_folders,
"Output file under wrong path, was '%s', should be '%s'" % (
self.out_filenames[0],
os.path.join(expected_folders[0],
os.path.basename(self.out_filenames[0])),
),
)
# Prettify generated XML
pretty_xml = XmlJob(xml_project, "fixturejob").output().decode("utf-8")
self.assertThat(
pretty_xml,
testtools.matchers.DocTestMatches(
expected_xml, doctest.ELLIPSIS | doctest.REPORT_NDIFF),
)