def load_secrets(self, filename, config_dict): if 'secretsProviders' in config_dict: secretsProviders = config_dict["secretsProviders"] if not isinstance(secretsProviders, list): error("c['secretsProviders'] must be a list") else: self.secretsProviders = secretsProviders
def load_builders(self, filename, config_dict): if 'builders' not in config_dict: return builders = config_dict['builders'] if not isinstance(builders, (list, tuple)): error("c['builders'] must be a list") return # convert all builder configs to BuilderConfig instances def mapper(b): if isinstance(b, BuilderConfig): return b elif isinstance(b, dict): return BuilderConfig(**b) else: error(f"{repr(b)} is not a builder config (in c['builders']") return None builders = [mapper(b) for b in builders] for builder in builders: if builder and os.path.isabs(builder.builddir): warnings.warn( (f"Absolute path '{builder.builddir}' for builder may cause mayhem. " "Perhaps you meant to specify workerbuilddir instead."), category=ConfigWarning, ) self.builders = builders
def load_caches(self, filename, config_dict): explicit = False if 'caches' in config_dict: explicit = True caches = config_dict['caches'] if not isinstance(caches, dict): error("c['caches'] must be a dictionary") else: for (name, value) in caches.items(): if not isinstance(value, int): error( f"value for cache size '{name}' must be an integer" ) return if value < 1: error( f"'{name}' cache size must be at least 1, got '{value}'" ) self.caches.update(caches) if 'buildCacheSize' in config_dict: if explicit: msg = "cannot specify c['caches'] and c['buildCacheSize']" error(msg) self.caches['Builds'] = config_dict['buildCacheSize'] if 'changeCacheSize' in config_dict: if explicit: msg = "cannot specify c['caches'] and c['changeCacheSize']" error(msg) self.caches['Changes'] = config_dict['changeCacheSize']
def load_metrics(self, filename, config_dict): # we don't try to validate metrics keys if 'metrics' in config_dict: metrics = config_dict["metrics"] if not isinstance(metrics, dict): error("c['metrics'] must be a dictionary") else: self.metrics = metrics
def mapper(b): if isinstance(b, BuilderConfig): return b elif isinstance(b, dict): return BuilderConfig(**b) else: error(f"{repr(b)} is not a builder config (in c['builders']") return None
def check_lock(lock): if isinstance(lock, locks.LockAccess): lock = lock.lockid if lock.name in lock_dict: if lock_dict[lock.name] is not lock: msg = f"Two locks share the same name, '{lock.name}'" error(msg) else: lock_dict[lock.name] = lock
def load_user_managers(self, filename, config_dict): if 'user_managers' not in config_dict: return user_managers = config_dict['user_managers'] msg = "c['user_managers'] must be a list of user managers" if not isinstance(user_managers, (list, tuple)): error(msg) return self.user_managers = user_managers
def load_validation(self, filename, config_dict): validation = config_dict.get("validation", {}) if not isinstance(validation, dict): error("c['validation'] must be a dictionary") else: unknown_keys = (set(validation.keys()) - set(self.validation.keys())) if unknown_keys: error( f"unrecognized validation key(s): {', '.join(unknown_keys)}" ) else: self.validation.update(validation)
def check_machines(self): seen_names = set() for mm in self.machines: if mm.name in seen_names: error(f"duplicate machine name '{mm.name}'") seen_names.add(mm.name) for w in self.workers: if w.machine_name is not None and w.machine_name not in seen_names: error( f"worker '{w.name}' uses unknown machine '{w.machine_name}'" )
def load_change_sources(self, filename, config_dict): change_source = config_dict.get('change_source', []) if isinstance(change_source, (list, tuple)): change_sources = change_source else: change_sources = [change_source] for s in change_sources: if not interfaces.IChangeSource.providedBy(s): msg = "c['change_source'] must be a list of change sources" error(msg) return self.change_sources = change_sources
def getDbUrlFromConfig(config_dict, throwErrors=True): if 'db' in config_dict: db = config_dict['db'] if set(db.keys()) - set(['db_url']) and throwErrors: error("unrecognized keys in c['db']") config_dict = db # we don't attempt to parse db URLs here - the engine strategy will do # so. if 'db_url' in config_dict: return config_dict['db_url'] return DEFAULT_DB_URL
def check_param_length(value, name, max_length): if isinstance(value, str) and len(value) > max_length: error(f"{name} '{value}' exceeds maximum length of {max_length}") qualified_name = f"{type(value).__module__}.{type(value).__name__}" if qualified_name == 'buildbot.process.properties.Interpolate': if value.args: interpolations = tuple([''] * len(value.args)) else: interpolations = {k: '' for k in value.interpolations} shortest_value = value.fmtstring % interpolations if len(shortest_value) > max_length: error( f"{name} '{value}' (shortest interpolation) exceeds maximum length of " f"{max_length}")
def load_mq(self, filename, config_dict): from buildbot.mq import connector # avoid circular imports if 'mq' in config_dict: self.mq.update(config_dict['mq']) classes = connector.MQConnector.classes typ = self.mq.get('type', 'simple') if typ not in classes: error(f"mq type '{typ}' is not known") return known_keys = classes[typ]['keys'] unk = set(self.mq.keys()) - known_keys - set(['type']) if unk: error(f"unrecognized keys in c['mq']: {', '.join(unk)}")
def check_schedulers(self): # don't perform this check in multiMaster mode if self.multiMaster: return all_buildernames = {b.name for b in self.builders} for s in self.schedulers.values(): builderNames = s.listBuilderNames() if interfaces.IRenderable.providedBy(builderNames): continue for n in builderNames: if interfaces.IRenderable.providedBy(n): continue if n not in all_buildernames: error(f"Unknown builder '{n}' in scheduler '{s.name}'")
def load_services(self, filename, config_dict): if 'services' not in config_dict: return self.services = {} for _service in config_dict['services']: if not isinstance(_service, util_service.BuildbotService): error(f"{type(_service)} object should be an instance of " "buildbot.util.service.BuildbotService") continue if _service.name in self.services: error(f'Duplicate service name {repr(_service.name)}') continue self.services[_service.name] = _service
def load_machines(self, filename, config_dict): if 'machines' not in config_dict: return machines = config_dict['machines'] msg = "c['machines'] must be a list of machines" if not isinstance(machines, (list, tuple)): error(msg) return for m in machines: if not interfaces.IMachine.providedBy(m): error(msg) return self.machines = machines
def copy_param(name, alt_key=None, check_type=None, check_type_name=None, can_be_callable=False): if name in config_dict: v = config_dict[name] elif alt_key and alt_key in config_dict: v = config_dict[alt_key] else: return if v is not None and check_type and not (isinstance( v, check_type) or (can_be_callable and callable(v))): error(f"c['{name}'] must be {check_type_name}") else: setattr(self, name, v)
def load_www(self, filename, config_dict): if 'www' not in config_dict: return www_cfg = config_dict['www'] allowed = { 'allowed_origins', 'auth', 'authz', 'avatar_methods', 'change_hook_auth', 'change_hook_dialects', 'cookie_expiration_time', 'custom_templates_dir', 'debug', 'default_page', 'json_cache_seconds', 'jsonp', 'logRotateLength', 'logfileName', 'maxRotatedFiles', 'plugins', 'port', 'rest_minimum_version', 'ui_default_config', 'versions', 'ws_ping_interval', 'graphql', } unknown = set(list(www_cfg)) - allowed if unknown: error( f"unknown www configuration parameter(s) {', '.join(unknown)}") versions = www_cfg.get('versions') if versions is not None: cleaned_versions = [] if not isinstance(versions, list): error('Invalid www configuration value of versions') else: for v in versions: if not isinstance(v, tuple) or len(v) < 2: error('Invalid www configuration value of versions') break cleaned_versions.append(v) www_cfg['versions'] = cleaned_versions cookie_expiration_time = www_cfg.get('cookie_expiration_time') if cookie_expiration_time is not None: if not isinstance(cookie_expiration_time, datetime.timedelta): error( 'Invalid www["cookie_expiration_time"] configuration should ' 'be a datetime.timedelta') self.www.update(www_cfg)
def loadFromDict(cls, config_dict, filename): # warning, all of this is loaded from a thread with capture_config_errors(raise_on_error=True): # check for unknown keys unknown_keys = set(config_dict.keys()) - cls._known_config_keys if unknown_keys: if len(unknown_keys) == 1: error( f'Unknown BuildmasterConfig key {unknown_keys.pop()}') else: error( f"Unknown BuildmasterConfig keys {', '.join(sorted(unknown_keys))}" ) # instantiate a new config object, which will apply defaults # automatically config = cls() # and defer the rest to sub-functions, for code clarity config.run_configurators(filename, config_dict) config.load_global(filename, config_dict) config.load_validation(filename, config_dict) config.load_db(filename, config_dict) config.load_mq(filename, config_dict) config.load_metrics(filename, config_dict) config.load_secrets(filename, config_dict) config.load_caches(filename, config_dict) config.load_schedulers(filename, config_dict) config.load_builders(filename, config_dict) config.load_workers(filename, config_dict) config.load_change_sources(filename, config_dict) config.load_machines(filename, config_dict) config.load_user_managers(filename, config_dict) config.load_www(filename, config_dict) config.load_services(filename, config_dict) # run some sanity checks config.check_single_master() config.check_schedulers() config.check_locks() config.check_builders() config.check_ports() config.check_machines() return config
def check_ports(self): ports = set() if self.protocols: for proto, options in self.protocols.items(): if proto == 'null': port = -1 else: port = options.get("port") if port is None: continue if isinstance(port, int): # Conversion needed to compare listenTCP and strports ports port = f"tcp:{port}" if port != -1 and port in ports: error("Some of ports in c['protocols'] duplicated") ports.add(port) if ports: return if self.workers: error("workers are configured, but c['protocols'] not")
def check_single_master(self): # check additional problems that are only valid in a single-master # installation if self.multiMaster: return if not self.workers: error("no workers are configured") if not self.builders: error("no builders are configured") # check that all builders are implemented on this master unscheduled_buildernames = {b.name for b in self.builders} for s in self.schedulers.values(): builderNames = s.listBuilderNames() if interfaces.IRenderable.providedBy(builderNames): unscheduled_buildernames.clear() else: for n in builderNames: if interfaces.IRenderable.providedBy(n): unscheduled_buildernames.clear() elif n in unscheduled_buildernames: unscheduled_buildernames.remove(n) if unscheduled_buildernames: names_str = ', '.join(unscheduled_buildernames) error(f"builder(s) {names_str} have no schedulers to drive them")
def _check_workers(workers, conf_key): if not isinstance(workers, (list, tuple)): error(f"{conf_key} must be a list") return False for worker in workers: if not interfaces.IWorker.providedBy(worker): msg = f"{conf_key} must be a list of Worker instances but there is {worker!r}" error(msg) return False def validate(workername): if workername in ("debug", "change", "status"): yield f"worker name {workername!r} is reserved" if not util_identifiers.ident_re.match(workername): yield f"worker name {workername!r} is not an identifier" if not workername: yield f"worker name {workername!r} cannot be an empty string" max_workername = 50 if len(workername) > max_workername: yield f"worker name {workername!r} is longer than {max_workername} characters" errors = list(validate(worker.workername)) for msg in errors: error(msg) if errors: return False return True
def load_schedulers(self, filename, config_dict): if 'schedulers' not in config_dict: return schedulers = config_dict['schedulers'] ok = True if not isinstance(schedulers, (list, tuple)): ok = False else: for s in schedulers: if not interfaces.IScheduler.providedBy(s): ok = False if not ok: msg = "c['schedulers'] must be a list of Scheduler instances" error(msg) # convert from list to dict, first looking for duplicates seen_names = set() for s in schedulers: if s.name in seen_names: error(f"scheduler name '{s.name}' used multiple times") seen_names.add(s.name) self.schedulers = dict((s.name, s) for s in schedulers)
def loadConfigDict(basedir, configFileName): if not os.path.isdir(basedir): raise ConfigErrors([f"basedir '{basedir}' does not exist"]) filename = os.path.join(basedir, configFileName) if not os.path.exists(filename): raise ConfigErrors([f"configuration file '{filename}' does not exist"]) try: with open(filename, "r", encoding='utf-8'): pass except IOError as e: raise ConfigErrors([ f"unable to open configuration file {repr(filename)}: {e}" ]) from e log.msg(f"Loading configuration from {repr(filename)}") # execute the config file localDict = { 'basedir': os.path.expanduser(basedir), '__file__': os.path.abspath(filename), } old_sys_path = sys.path[:] sys.path.append(basedir) try: try: execfile(filename, localDict) except ConfigErrors: raise except SyntaxError: error( f"encountered a SyntaxError while parsing config file:\n{traceback.format_exc()} ", always_raise=True) except Exception: log.err(failure.Failure(), 'error while parsing config file:') error( f"error while parsing config file: {sys.exc_info()[1]} (traceback in logfile)", always_raise=True) finally: sys.path[:] = old_sys_path if 'BuildmasterConfig' not in localDict: error( f"Configuration file {repr(filename)} does not define 'BuildmasterConfig'", always_raise=True, ) return filename, localDict['BuildmasterConfig']
def check_builders(self): # look both for duplicate builder names, and for builders pointing # to unknown workers workernames = {w.workername for w in self.workers} seen_names = set() seen_builddirs = set() for b in self.builders: unknowns = set(b.workernames) - workernames if unknowns: error(f"builder '{b.name}' uses unknown workers " f"{', '.join(repr(u) for u in unknowns)}") if b.name in seen_names: error(f"duplicate builder name '{b.name}'") seen_names.add(b.name) if b.builddir in seen_builddirs: error(f"duplicate builder builddir '{b.builddir}'") seen_builddirs.add(b.builddir)
def load_global(self, filename, config_dict): def copy_param(name, alt_key=None, check_type=None, check_type_name=None, can_be_callable=False): if name in config_dict: v = config_dict[name] elif alt_key and alt_key in config_dict: v = config_dict[alt_key] else: return if v is not None and check_type and not (isinstance( v, check_type) or (can_be_callable and callable(v))): error(f"c['{name}'] must be {check_type_name}") else: setattr(self, name, v) def copy_int_param(name, alt_key=None): copy_param(name, alt_key=alt_key, check_type=int, check_type_name='an int') def copy_str_param(name, alt_key=None): copy_param(name, alt_key=alt_key, check_type=(str, ), check_type_name='a string') copy_str_param('title', alt_key='projectName') max_title_len = 18 if len(self.title) > max_title_len: # Warn if the title length limiting logic in www/base/src/app/app.route.js # would hide the title. warnings.warn('WARNING: Title is too long to be displayed. ' + '"Buildbot" will be used instead.', category=ConfigWarning) copy_str_param('titleURL', alt_key='projectURL') copy_str_param('buildbotURL') def copy_str_or_callable_param(name, alt_key=None): copy_param(name, alt_key=alt_key, check_type=(str, ), check_type_name='a string or callable', can_be_callable=True) if "buildbotNetUsageData" not in config_dict: if get_is_in_unit_tests(): self.buildbotNetUsageData = None else: warnings.warn( '`buildbotNetUsageData` is not configured and defaults to basic.\n' 'This parameter helps the buildbot development team to understand' ' the installation base.\n' 'No personal information is collected.\n' 'Only installation software version info and plugin usage is sent.\n' 'You can `opt-out` by setting this variable to None.\n' 'Or `opt-in` for more information by setting it to "full".\n', category=ConfigWarning) copy_str_or_callable_param('buildbotNetUsageData') copy_int_param('changeHorizon') copy_int_param('logCompressionLimit') self.logCompressionMethod = config_dict.get('logCompressionMethod', 'gz') if self.logCompressionMethod not in ('raw', 'bz2', 'gz', 'lz4'): error( "c['logCompressionMethod'] must be 'raw', 'bz2', 'gz' or 'lz4'" ) if self.logCompressionMethod == "lz4": try: import lz4 # pylint: disable=import-outside-toplevel [lz4] except ImportError: error("To set c['logCompressionMethod'] to 'lz4' " "you must install the lz4 library ('pip install lz4')") copy_int_param('logMaxSize') copy_int_param('logMaxTailSize') copy_param('logEncoding') properties = config_dict.get('properties', {}) if not isinstance(properties, dict): error("c['properties'] must be a dictionary") else: self.properties.update(properties, filename) collapseRequests = config_dict.get('collapseRequests') if (collapseRequests not in (None, True, False) and not callable(collapseRequests)): error("collapseRequests must be a callable, True, or False") else: self.collapseRequests = collapseRequests codebaseGenerator = config_dict.get('codebaseGenerator') if (codebaseGenerator is not None and not callable(codebaseGenerator)): error( "codebaseGenerator must be a callable accepting a dict and returning a str" ) else: self.codebaseGenerator = codebaseGenerator prioritizeBuilders = config_dict.get('prioritizeBuilders') if prioritizeBuilders is not None and not callable(prioritizeBuilders): error("prioritizeBuilders must be a callable") else: self.prioritizeBuilders = prioritizeBuilders protocols = config_dict.get('protocols', {}) if isinstance(protocols, dict): for proto, options in protocols.items(): if not isinstance(proto, str): error("c['protocols'] keys must be strings") if not isinstance(options, dict): error(f"c['protocols']['{proto}'] must be a dict") return if proto == "wamp": self.check_wamp_proto(options) else: error("c['protocols'] must be dict") return self.protocols = protocols if 'multiMaster' in config_dict: self.multiMaster = config_dict["multiMaster"] if 'debugPassword' in config_dict: log.msg("the 'debugPassword' parameter is unused and " "can be removed from the configuration file") if 'manhole' in config_dict: # we don't check that this is a manhole instance, since that # requires importing buildbot.manhole for every user, and currently # that will fail if cryptography isn't installed self.manhole = config_dict['manhole'] if 'revlink' in config_dict: revlink = config_dict['revlink'] if not callable(revlink): error("revlink must be a callable") else: self.revlink = revlink
def test_error_raises(self): with self.assertRaises(ConfigErrors) as e: error("message") self.assertEqual(e.exception.errors, ["message"])
def test_error_no_raise(self): with capture_config_errors() as errors: error("message") self.assertEqual(errors.errors, ["message"])
def __init__(self, name=None, workername=None, workernames=None, builddir=None, workerbuilddir=None, factory=None, tags=None, nextWorker=None, nextBuild=None, locks=None, env=None, properties=None, collapseRequests=None, description=None, canStartBuild=None, defaultProperties=None ): # name is required, and can't start with '_' if not name or type(name) not in (bytes, str): error("builder's name is required") name = '<unknown>' elif name[0] == '_' and name not in RESERVED_UNDERSCORE_NAMES: error(f"builder names must not start with an underscore: '{name}'") try: self.name = bytes2unicode(name, encoding="ascii") except UnicodeDecodeError: error("builder names must be unicode or ASCII") # factory is required if factory is None: error(f"builder '{name}' has no factory") from buildbot.process.factory import BuildFactory if factory is not None and not isinstance(factory, BuildFactory): error(f"builder '{name}'s factory is not a BuildFactory instance") self.factory = factory # workernames can be a single worker name or a list, and should also # include workername, if given if isinstance(workernames, str): workernames = [workernames] if workernames: if not isinstance(workernames, list): error(f"builder '{name}': workernames must be a list or a string") else: workernames = [] if workername: if not isinstance(workername, str): error(f"builder '{name}': workername must be a string but it is {repr(workername)}") workernames = workernames + [workername] if not workernames: error(f"builder '{name}': at least one workername is required") self.workernames = workernames # builddir defaults to name if builddir is None: builddir = safeTranslate(name) builddir = bytes2unicode(builddir) self.builddir = builddir # workerbuilddir defaults to builddir if workerbuilddir is None: workerbuilddir = builddir self.workerbuilddir = workerbuilddir # remainder are optional if tags: if not isinstance(tags, list): error(f"builder '{name}': tags must be a list") bad_tags = any((tag for tag in tags if not isinstance(tag, str))) if bad_tags: error(f"builder '{name}': tags list contains something that is not a string") if len(tags) != len(set(tags)): dupes = " ".join({x for x in tags if tags.count(x) > 1}) error(f"builder '{name}': tags list contains duplicate tags: {dupes}") else: tags = [] self.tags = tags self.nextWorker = nextWorker if nextWorker and not callable(nextWorker): error('nextWorker must be a callable') self.nextBuild = nextBuild if nextBuild and not callable(nextBuild): error('nextBuild must be a callable') self.canStartBuild = canStartBuild if canStartBuild and not callable(canStartBuild): error('canStartBuild must be a callable') self.locks = locks or [] self.env = env or {} if not isinstance(self.env, dict): error("builder's env must be a dictionary") self.properties = properties or {} for property_name in self.properties: check_param_length(property_name, f'Builder {self.name} property', Model.property_name_length) self.defaultProperties = defaultProperties or {} for property_name in self.defaultProperties: check_param_length(property_name, f'Builder {self.name} default property', Model.property_name_length) self.collapseRequests = collapseRequests self.description = description