def CMDbuilders(parser, args):
"""Prints json-formatted list of builders given a path to cq.cfg file.
The output is a dictionary in the following format:
{
'master_name': [
'builder_name',
'another_builder'
],
'another_master': [
'third_builder'
]
}
"""
_, args = parser.parse_args(args)
if len(args) != 1:
parser.error('Expected a single path to CQ config. Got: %s' %
' '.join(args))
with open(args[0]) as config_file:
cq_config = config_file.read()
config = cq_pb2.Config()
text_format.Merge(cq_config, config)
masters = {}
if config.HasField('verifiers') and config.verifiers.HasField('try_job'):
for bucket in config.verifiers.try_job.buckets:
masters.setdefault(bucket.name, [])
for builder in bucket.builders:
if not builder.HasField('experiment_percentage'):
masters[bucket.name].append(builder.name)
print json.dumps(masters)
def test_has_field(self):
config = cq_pb2.Config()
self.assertFalse(validate_config._HasField(config, 'version'))
config.version = 1
self.assertTrue(validate_config._HasField(config, 'version'))
self.assertFalse(
validate_config._HasField(config, 'rietveld.project_bases'))
config.rietveld.project_bases.append('foo://bar')
self.assertTrue(
validate_config._HasField(config, 'rietveld.project_bases'))
self.assertFalse(
validate_config._HasField(config, 'verifiers.try_job.buckets'))
self.assertFalse(
validate_config._HasField(config,
'verifiers.try_job.buckets.name'))
bucket = config.verifiers.try_job.buckets.add()
bucket.name = 'tryserver.chromium.linux'
self.assertTrue(
validate_config._HasField(config, 'verifiers.try_job.buckets'))
self.assertTrue(
validate_config._HasField(config,
'verifiers.try_job.buckets.name'))
config.verifiers.try_job.buckets.add()
self.assertFalse(
validate_config._HasField(config,
'verifiers.try_job.buckets.name'))
def IsValid(cq_config):
"""Validates a CQ config and prints errors/warnings to the screen.
Args:
cq_config (string): Unparsed text format of the CQ config proto.
Returns:
True if the config is valid.
"""
try:
config = cq_pb2.Config()
protobuf.text_format.Merge(cq_config, config)
except protobuf.text_format.ParseError as e:
logging.error('Failed to parse config as protobuf:\n%s', e)
return False
if _HasField(config, 'gerrit'):
if _HasField(config, 'rietveld'):
logging.error('gerrit and rietveld are not supported at the same time.')
return False
# TODO(tandrii): validate gerrit.
required_fields = REQUIRED_FIELDS + ['gerrit.cq_verified_label']
if _HasField(config, 'verifiers.reviewer_lgtm'):
logging.error('reviewer_lgtm verifier is not supported with Gerrit.')
return False
elif _HasField(config, 'rietveld'):
required_fields = REQUIRED_FIELDS + ['rietveld.url']
else:
logging.error('either rietveld gerrit are required fields.')
return False
for fname in required_fields:
if not _HasField(config, fname):
logging.error('%s is a required field', fname)
return False
for fname in LEGACY_FIELDS:
if _HasField(config, fname):
logging.warn('%s is a legacy field', fname)
for base in config.rietveld.project_bases:
try:
re.compile(base)
except re.error:
logging.error('failed to parse "%s" in project_bases as a regexp', base)
return False
# TODO(sergiyb): For each field, check valid values depending on its
# semantics, e.g. email addresses, regular expressions etc.
return True
def get_master_builder_map(
config_path, include_experimental=True, include_triggered=True):
"""Returns a map of master -> [builders] from cq config."""
with open(config_path) as config_file:
cq_config = config_file.read()
config = cq_pb2.Config()
text_format.Merge(cq_config, config)
masters = {}
if config.HasField('verifiers') and config.verifiers.HasField('try_job'):
for bucket in config.verifiers.try_job.buckets:
masters.setdefault(bucket.name, [])
for builder in bucket.builders:
if (not include_experimental and
builder.HasField('experiment_percentage')):
continue
if (not include_triggered and
builder.HasField('triggered_by')):
continue
masters[bucket.name].append(builder.name)
return masters
def IsValid(cq_config):
"""Validates a CQ config and prints errors/warnings to the screen.
Args:
cq_config (string): Unparsed text format of the CQ config proto.
Returns:
True if the config is valid.
"""
try:
config = cq_pb2.Config()
protobuf.text_format.Merge(cq_config, config)
except protobuf.text_format.ParseError as e:
logging.error('Failed to parse config as protobuf:\n%s', e)
return False
for fname in REQUIRED_FIELDS:
if not _HasField(config, fname):
logging.error('%s is a required field', fname)
return False
for fname in LEGACY_FIELDS:
if _HasField(config, fname):
logging.warn('%s is a legacy field', fname)
for base in config.rietveld.project_bases:
try:
re.compile(base)
except re.error:
logging.error('failed to parse "%s" in project_bases as a regexp', base)
return False
# TODO(sergiyb): For each field, check valid values depending on its
# semantics, e.g. email addresses, regular expressions etc.
return True
def CMDvalidate(parser, args):
"""Validates a CQ config, returns 0 on valid config.
BUGS: this doesn't do semantic validation, only verifies validity of protobuf.
But don't worry - bad cq.cfg won't cause outages, luci-config service will
not accept them, will send warning email, and continue using previous
version.
"""
_, args = parser.parse_args(args)
if len(args) != 1:
parser.error('Expected a single path to CQ config. Got: %s' %
' '.join(args))
config = cq_pb2.Config()
try:
with open(args[0]) as config_file:
text_config = config_file.read()
text_format.Merge(text_config, config)
# TODO(tandrii): provide an option to actually validate semantics of CQ
# config.
return 0
except text_format.ParseError as e:
print 'failed to parse cq.cfg: %s' % e
return 1