def parse_fp(self, fp):
# wrap provided file streams to ensure correct encoding used
data = local_yaml.load(utils.wrap_stream(fp), search_path=self.path)
if data:
if not isinstance(data, list):
raise JenkinsJobsException(
"The topmost collection in file '{fname}' must be a list,"
" not a {cls}".format(fname=getattr(fp, 'name', fp),
cls=type(data)))
for item in data:
cls, dfn = next(iter(item.items()))
group = self.data.get(cls, {})
if len(item.items()) > 1:
n = None
for k, v in item.items():
if k == "name":
n = v
break
# Syntax error
raise JenkinsJobsException(
"Syntax error, for item "
"named '{0}'. Missing indent?".format(n))
name = dfn['name']
if name in group:
self._handle_dups("Duplicate entry found in '{0}: '{1}' "
"already defined".format(fp.name, name))
group[name] = dfn
self.data[cls] = group
def parse_fp(self, fp):
data = local_yaml.load(fp, search_path=self.path)
if data:
if not isinstance(data, list):
raise JenkinsJobsException(
"The topmost collection in file '{fname}' must be a list,"
" not a {cls}".format(fname=getattr(fp, 'name', fp),
cls=type(data)))
for item in data:
cls, dfn = next(iter(item.items()))
group = self.data.get(cls, {})
if len(item.items()) > 1:
n = None
for k, v in item.items():
if k == "name":
n = v
break
# Syntax error
raise JenkinsJobsException("Syntax error, for item "
"named '{0}'. Missing indent?"
.format(n))
name = dfn['name']
if name in group:
self._handle_dups("Duplicate entry found in '{0}: '{1}' "
"already defined".format(fp.name, name))
group[name] = dfn
self.data[cls] = group
def _parse_fp(self, fp):
# wrap provided file streams to ensure correct encoding used
data = local_yaml.load(utils.wrap_stream(fp), search_path=self.path)
if data:
if not isinstance(data, list):
raise JenkinsJobsException(
"The topmost collection in file '{fname}' must be a list,"
" not a {cls}".format(fname=getattr(fp, 'name', fp),
cls=type(data)))
for item in data:
cls, dfn = next(iter(item.items()))
group = self.data.get(cls, {})
if len(item.items()) > 1:
n = None
for k, v in item.items():
if k == "name":
n = v
break
# Syntax error
raise JenkinsJobsException("Syntax error, for item "
"named '{0}'. Missing indent?"
.format(n))
# allow any entry to specify an id that can also be used
_id = dfn.get('id', dfn['name'])
if _id in group:
self._handle_dups(
"Duplicate entry found in '{0}: '{1}' already "
"defined".format(fp.name, _id))
group[_id] = dfn
self.data[cls] = group
def _parse_fp(self, fp):
# wrap provided file streams to ensure correct encoding used
data = local_yaml.load(utils.wrap_stream(fp),
self.jjb_config.yamlparser['retain_anchors'],
search_path=self.path)
if data:
if not isinstance(data, list):
raise JenkinsJobsException(
"The topmost collection in file '{fname}' must be a list,"
" not a {cls}".format(fname=getattr(fp, 'name', fp),
cls=type(data)))
for item in data:
cls, dfn = next(iter(item.items()))
group = self.data.get(cls, {})
if len(item.items()) > 1:
n = None
for k, v in item.items():
if k == "name":
n = v
break
# Syntax error
raise JenkinsJobsException(
"Syntax error, for item "
"named '{0}'. Missing indent?".format(n))
# allow any entry to specify an id that can also be used
_id = dfn.get('id', dfn['name'])
if _id in group:
self._handle_dups(
"Duplicate entry found in '{0}: '{1}' already "
"defined".format(fp.name, _id))
group[_id] = dfn
self.data[cls] = group
def validate_jobs():
"""Minimal YAML file validation."""
count = 0
errors = False
print("Validating YAML files")
print("=====================")
for job_file in glob.glob('jenkins/jobs/*.yaml'):
jobs = local_yaml.load(io.open(job_file, 'r', encoding='utf-8'))
for item in jobs:
if 'builder' in item:
schema = BUILDER
entry = item['builder']
elif 'job' in item:
schema = JOB
entry = item['job']
elif 'job-group' in item:
schema = JOB_GROUP
entry = item['job-group']
elif 'job-template' in item:
schema = JOB_TEMPLATE
entry = item['job-template']
elif 'project' in item:
schema = PROJECT
entry = item['project']
elif 'publisher' in item:
schema = PUBLISHER
entry = item['publisher']
elif 'wrapper' in item:
continue
elif 'defaults' in item:
continue
else:
print("Unknown entry in file %s" % job_file)
print(item)
try:
schema(entry)
except Exception as e:
print("Failure: %s" % e)
print("Failure in file %s" % job_file)
print("Failure parsing item:")
print(item)
count += 1
errors = True
# NOTE(pabelanger): Make sure console-log is our last publisher
# defined. We use the publisher to upload logs from zuul-launcher.
result = _check_console_log_publisher(schema, entry)
result += _check_tox_builder(schema, entry)
if result:
print(job_file)
count += result
errors = True
print("%d errors found validating YAML files in jenkins/jobs/*.yaml.\n" %
count)
return errors
def validate_jobs():
"""Minimal YAML file validation."""
count = 0
errors = False
print("Validating YAML files")
print("=====================")
for job_file in glob.glob('jenkins/jobs/*.yaml'):
jobs = local_yaml.load(io.open(job_file, 'r', encoding='utf-8'))
for item in jobs:
if 'builder' in item:
schema = BUILDER
entry = item['builder']
elif 'job' in item:
schema = JOB
entry = item['job']
elif 'job-group' in item:
schema = JOB_GROUP
entry = item['job-group']
elif 'job-template' in item:
schema = JOB_TEMPLATE
entry = item['job-template']
elif 'project' in item:
schema = PROJECT
entry = item['project']
elif 'publisher' in item:
schema = PUBLISHER
entry = item['publisher']
elif 'wrapper' in item:
continue
elif 'defaults' in item:
continue
else:
print("Unknown entry in file %s" % job_file)
print(item)
try:
schema(entry)
except Exception as e:
print("Failure: %s" % e)
print("Failure in file %s" % job_file)
print("Failure parsing item:")
print(item)
count += 1
errors = True
# NOTE(pabelanger): Make sure console-log is our last publisher
# defined. We use the publisher to upload logs from zuul-launcher.
result = _check_console_log_publisher(schema, entry)
result += _check_tox_builder(schema, entry)
if result:
print(job_file)
count += result
errors = True
print("%d errors found validating YAML files in jenkins/jobs/*.yaml.\n" % count)
return errors
def _read_content(self):
# Read XML content, assuming it is unicode encoded
xml_filepath = os.path.join(self.fixtures_path, self.out_filename)
xml_content = u"%s" % codecs.open(xml_filepath, 'r', 'utf-8').read()
yaml_filepath = os.path.join(self.fixtures_path, self.in_filename)
with file(yaml_filepath, 'r') as yaml_file:
yaml_content = yaml.load(yaml_file)
return (yaml_content, xml_content)
def _read_content(self):
# Read XML content, assuming it is unicode encoded
xml_filepath = os.path.join(self.fixtures_path, self.out_filename)
xml_content = u"%s" % codecs.open(xml_filepath, 'r', 'utf-8').read()
yaml_filepath = os.path.join(self.fixtures_path, self.in_filename)
with file(yaml_filepath, 'r') as yaml_file:
yaml_content = yaml.load(yaml_file)
return (yaml_content, xml_content)
def validate_service_types():
print("Validating Service Types")
print("========================")
count = 0
# Load the current service-type-authority data
service_types = ServiceTypeValidator()
# Load the project job definitions
with io.open('jenkins/jobs/projects.yaml', 'r', encoding='utf-8') as f:
file_contents = local_yaml.load(f.read())
for item in file_contents:
project = item.get('project', {})
for job in project.get('jobs', []):
for api_job in _API_JOBS:
if api_job not in job:
continue
try:
proj_data = service_types.get_data_for_project(
project['name'])
except ValueError:
print(
'ERROR: Found service type reference "{}" for {} in {} '
'but not in authority list {}'.format(
job[api_job]['service'], api_job, project['name'],
service_types.url))
count += 1
else:
actual = job[api_job]['service']
expected = proj_data['service_type']
if actual != expected:
print('ERROR: Found service "{}" for {} '
'in {} but expected "{}"'.format(
job[api_job]['service'], api_job,
project['name'], expected))
count += 1
print('Found {} errors in service type settings '
'in jenkins/jobs/projects.yaml\n'.format(count))
return count
def _read_yaml_content(self, filename):
with io.open(filename, 'r', encoding='utf-8') as yaml_file:
yaml_content = yaml.load(yaml_file)
return yaml_content
def _read_yaml_content(self, filename):
with open(filename, 'r') as yaml_file:
yaml_content = yaml.load(yaml_file)
return yaml_content
def _read_yaml_content(self):
yaml_filepath = os.path.join(self.fixtures_path, self.in_filename)
with open(yaml_filepath, 'r') as yaml_file:
yaml_content = yaml.load(yaml_file)
return yaml_content
def _read_yaml_content(self, filename):
with open(filename, 'r') as yaml_file:
yaml_content = yaml.load(yaml_file)
return yaml_content
def _read_yaml_content(self):
yaml_filepath = os.path.join(self.fixtures_path, self.in_filename)
with open(yaml_filepath, 'r') as yaml_file:
yaml_content = yaml.load(yaml_file)
return yaml_content
