def loadBuilderConfig(c, is_test_mode_enabled=False, master_prefix_path='./'): config = json.load(open(os.path.join(master_prefix_path, 'config.json'))) if is_test_mode_enabled: passwords = {} else: passwords = json.load(open(os.path.join(master_prefix_path, 'passwords.json'))) checkWorkersAndBuildersForConsistency(config, config['workers'], config['builders']) checkValidSchedulers(config, config['schedulers']) c['workers'] = [Worker(worker['name'], passwords.get(worker['name'], 'password'), max_builds=worker.get('max_builds', 1)) for worker in config['workers']] if is_test_mode_enabled: c['workers'].append(Worker('local-worker', 'password', max_builds=1)) c['builders'] = [] for builder in config['builders']: builder['tags'] = getTagsForBuilder(builder) factory = globals()[builder['factory']] builder['description'] = builder.pop('shortname') if 'icon' in builder: del builder['icon'] factorykwargs = {} for key in ['platform', 'configuration', 'architectures', 'triggers', 'remotes', 'additionalArguments', 'runTests', 'triggered_by']: value = builder.pop(key, None) if value: factorykwargs[key] = value builder['factory'] = factory(**factorykwargs) if is_test_mode_enabled: builder['workernames'].append('local-worker') c['builders'].append(builder) c['prioritizeBuilders'] = prioritizeBuilders c['schedulers'] = [] for scheduler in config['schedulers']: schedulerClassName = scheduler.pop('type') schedulerClass = globals()[schedulerClassName] if (schedulerClassName == 'Try_Userpass'): # FIXME: Read the credentials from local file on disk. scheduler['userpass'] = [(os.getenv('BUILDBOT_TRY_USERNAME', 'sampleuser'), os.getenv('BUILDBOT_TRY_PASSWORD', 'samplepass'))] c['schedulers'].append(schedulerClass(**scheduler)) if is_test_mode_enabled: forceScheduler = ForceScheduler( name="force_build", buttonName="Force Build", builderNames=[str(builder['name']) for builder in config['builders']], # Disable default enabled input fields: branch, repository, project, additional properties codebases=[CodebaseParameter("", revision=FixedParameter(name="revision", default=""), repository=FixedParameter(name="repository", default=""), project=FixedParameter(name="project", default=""), branch=FixedParameter(name="branch", default=""))], # Add custom properties needed properties=[StringParameter(name="patch_id", label="Patch attachment id number (not bug number)", required=True, maxsize=7), StringParameter(name="ews_revision", label="WebKit git sha1 hash to checkout before trying patch (optional)", required=False, maxsize=40)], ) c['schedulers'].append(forceScheduler)
def loadBuilderConfig(c, is_test_mode_enabled=False, master_prefix_path='./'): config = json.load(open(os.path.join(master_prefix_path, 'config.json'))) use_localhost_worker = is_test_mode_enabled if is_test_mode_enabled: passwords = {} else: passwords = json.load( open(os.path.join(master_prefix_path, 'passwords.json'))) checkWorkersAndBuildersForConsistency(config, config['workers'], config['builders']) checkValidSchedulers(config, config['schedulers']) c['workers'] = [ Worker(worker['name'], passwords.get(worker['name'], 'password'), max_builds=worker.get('max_builds', 1)) for worker in config['workers'] ] if use_localhost_worker: c['workers'].append(Worker('local-worker', 'password', max_builds=1)) c['builders'] = [] for builder in config['builders']: builder['tags'] = getTagsForBuilder(builder) factory = globals()[builder['factory']] builder['description'] = builder.pop('shortname') if 'icon' in builder: del builder['icon'] factorykwargs = {} for key in [ 'platform', 'configuration', 'architectures', 'triggers', 'remotes', 'additionalArguments', 'runTests', 'triggered_by' ]: value = builder.pop(key, None) if value: factorykwargs[key] = value builder['factory'] = factory(**factorykwargs) if use_localhost_worker: builder['workernames'].append('local-worker') c['builders'].append(builder) c['prioritizeBuilders'] = prioritizeBuilders c['schedulers'] = [] for scheduler in config['schedulers']: schedulerClassName = scheduler.pop('type') schedulerClass = globals()[schedulerClassName] if (schedulerClassName == 'Try_Userpass'): # FIXME: Read the credentials from local file on disk. scheduler['userpass'] = [(os.getenv('BUILDBOT_TRY_USERNAME', 'sampleuser'), os.getenv('BUILDBOT_TRY_PASSWORD', 'samplepass'))] c['schedulers'].append(schedulerClass(**scheduler))
def loadBuilderConfig(c, use_localhost_worker=False, master_prefix_path='./'): config = json.load(open(os.path.join(master_prefix_path, 'config.json'))) passwords = json.load( open(os.path.join(master_prefix_path, 'passwords.json'))) checkWorkersAndBuildersForConsistency(config['workers'], config['builders']) c['workers'] = [ Worker(worker['name'], passwords.get(worker['name'], 'password')) for worker in config['workers'] ] if use_localhost_worker: c['workers'].append(Worker('local-worker', 'password', max_builds=2)) c['builders'] = [] for builder in config['builders']: builder['tags'] = getTagsForBuilder(builder) factory = globals()[builder['factory']] factorykwargs = {} for key in [ "platform", "configuration", "architectures", "triggers", "additionalArguments" ]: value = builder.pop(key, None) if value: factorykwargs[key] = value builder["factory"] = factory(**factorykwargs) if use_localhost_worker: builder['workernames'].append("local-worker") c['builders'].append(builder) c['schedulers'] = [] for scheduler in config['schedulers']: schedulerClassName = scheduler.pop('type') schedulerClass = globals()[schedulerClassName] # Python 2.6 can't handle unicode keys as keyword arguments: # http://bugs.python.org/issue2646. Modern versions of json return # unicode strings from json.load, so we map all keys to str objects. scheduler = dict( map( lambda key_value_pair: (str(key_value_pair[0]), key_value_pair[1]), scheduler.items())) if (schedulerClassName == 'Try_Userpass'): # FIXME: Read the credentials from local file on disk. scheduler['userpass'] = [('sampleuser', 'samplepass')] c['schedulers'].append(schedulerClass(**scheduler)) force_scheduler = ForceScheduler( name='force-{0}'.format(scheduler['name']), builderNames=scheduler['builderNames'], properties=[WorkerChoiceParameter()]) c['schedulers'].append(force_scheduler)
def workerConfiguration(): """Create worker configuration for use in BuildBot configuration""" configuration = [] for credentials in WORKER_CREDENTIALS: configuration.append( Worker(credentials["name"], credentials["password"])) return configuration
def masterConfig(): c = {} from buildbot.worker import Worker from buildbot.config import BuilderConfig from buildbot.process.buildstep import BuildStep from buildbot.process.factory import BuildFactory from buildbot.process import results class MyBuildStep(BuildStep): def start(self): self.finished(results.SUCCESS) c['workers'] = [Worker("local1", "localpw")] c['protocols'] = {'pb': {'port': 'tcp:0'}} c['change_source'] = [] c['schedulers'] = [] # filled in above f1 = BuildFactory() f1.addStep(MyBuildStep(name='one')) f1.addStep(MyBuildStep(name='two')) c['builders'] = [ BuilderConfig(name="a", workernames=["local1"], factory=f1), ] c['status'] = [] c['title'] = "test" c['titleURL'] = "test" c['buildbotURL'] = "http://localhost:8010/" c['db'] = {'db_url': "sqlite:///state.sqlite"} c['mq'] = {'debug': True} # test wants to influence the config, but we still return a new config each time c.update(BuildmasterConfig) return c
def loadBuilderConfig(c): config = json.load(open('config.json')) passwords = json.load(open('passwords.json')) checkWorkersAndBuildersForConsistency(config['workers'], config['builders']) c['workers'] = [ Worker(worker['name'], passwords.get(worker['name'], 'password')) for worker in config['workers'] ] c['builders'] = [] for builder in config['builders']: builder['tags'] = getTagsForBuilder(builder) factory = globals()[builder['factory']] builder['factory'] = factory() del builder['platform'] if 'configuration' in builder: del builder['configuration'] c['builders'].append(builder) c['schedulers'] = [] for scheduler in config['schedulers']: schedulerType = globals()[scheduler.pop('type')] # Python 2.6 can't handle unicode keys as keyword arguments: # http://bugs.python.org/issue2646. Modern versions of json return # unicode strings from json.load, so we map all keys to str objects. scheduler = dict( map( lambda key_value_pair: (str(key_value_pair[0]), key_value_pair[1]), scheduler.items())) c['schedulers'].append(schedulerType(**scheduler))
def loadBuilderConfig(c, use_localhost_worker=False, master_prefix_path='./'): config = json.load(open(os.path.join(master_prefix_path, 'config.json'))) passwords = json.load( open(os.path.join(master_prefix_path, 'passwords.json'))) checkWorkersAndBuildersForConsistency(config['workers'], config['builders']) c['workers'] = [ Worker(worker['name'], passwords.get(worker['name'], 'password')) for worker in config['workers'] ] if use_localhost_worker: c['workers'].append(Worker('local-worker', 'password', max_builds=2)) c['builders'] = [] for builder in config['builders']: builder['tags'] = getTagsForBuilder(builder) factory = globals()[builder['factory']] builder['factory'] = factory() del builder['platform'] if 'configuration' in builder: del builder['configuration'] if use_localhost_worker: builder['workernames'].append("local-worker") c['builders'].append(builder) c['schedulers'] = [] for scheduler in config['schedulers']: schedulerType = globals()[scheduler.pop('type')] # Python 2.6 can't handle unicode keys as keyword arguments: # http://bugs.python.org/issue2646. Modern versions of json return # unicode strings from json.load, so we map all keys to str objects. scheduler = dict( map( lambda key_value_pair: (str(key_value_pair[0]), key_value_pair[1]), scheduler.items())) c['schedulers'].append(schedulerType(**scheduler)) force_scheduler = ForceScheduler( name='force-{0}'.format(scheduler['name']), builderNames=scheduler['builderNames'], properties=[WorkerChoiceParameter()]) c['schedulers'].append(force_scheduler)
def loadBuilderConfig(c, is_test_mode_enabled=False, master_prefix_path='./'): config = json.load(open(os.path.join(master_prefix_path, 'config.json'))) use_localhost_worker = is_test_mode_enabled if is_test_mode_enabled: passwords = {} else: passwords = json.load(open(os.path.join(master_prefix_path, 'passwords.json'))) checkWorkersAndBuildersForConsistency(config, config['workers'], config['builders']) checkValidSchedulers(config, config['schedulers']) c['workers'] = [Worker(worker['name'], passwords.get(worker['name'], 'password'), max_builds=worker.get('max_builds', 1)) for worker in config['workers']] if use_localhost_worker: c['workers'].append(Worker('local-worker', 'password', max_builds=1)) c['builders'] = [] for builder in config['builders']: builder['tags'] = getTagsForBuilder(builder) factory = globals()[builder['factory']] builder['description'] = builder.pop('shortname') factorykwargs = {} for key in ['platform', 'configuration', 'architectures', 'triggers', 'additionalArguments']: value = builder.pop(key, None) if value: factorykwargs[key] = value builder['factory'] = factory(**factorykwargs) if use_localhost_worker: builder['workernames'].append('local-worker') c['builders'].append(builder) c['schedulers'] = [] for scheduler in config['schedulers']: schedulerClassName = scheduler.pop('type') schedulerClass = globals()[schedulerClassName] # Python 2.6 can't handle unicode keys as keyword arguments: # http://bugs.python.org/issue2646. Modern versions of json return # unicode strings from json.load, so we map all keys to str objects. scheduler = dict(map(lambda key_value_pair: (str(key_value_pair[0]), key_value_pair[1]), scheduler.items())) if (schedulerClassName == 'Try_Userpass'): # FIXME: Read the credentials from local file on disk. scheduler['userpass'] = [(os.getenv('BUILDBOT_TRY_USERNAME', 'sampleuser'), os.getenv('BUILDBOT_TRY_PASSWORD', 'samplepass'))] c['schedulers'].append(schedulerClass(**scheduler))
def get_slaves(): import private return [ Worker('unassigned', private.slave_passwords['unassigned'], max_builds=1, properties={'parallel': 0}), Worker('expl0it3r-windows', private.slave_passwords['expl0it3r-windows'], max_builds=1, properties={'parallel': 4}), Worker('expl0it3r-raspbian-armhf', private.slave_passwords['expl0it3r-raspbian-armhf'], properties={'parallel': 3}), Worker('macos-64', private.slave_passwords['macos-64'], max_builds=1, properties={'parallel': 3}), Worker('macos-arm64', private.slave_passwords['macos-arm64'], max_builds=1, properties={'parallel': 3}), Worker('binary1248-debian', private.slave_passwords['binary1248-debian'], max_builds=1, properties={'parallel': 3}), Worker('binary1248-freebsd-64', private.slave_passwords['binary1248-freebsd-64'], max_builds=1, properties={'parallel': 1}), Worker('binary1248-windows', private.slave_passwords['binary1248-windows'], max_builds=1, properties={'parallel': 8}), Worker('binary1248-android', private.slave_passwords['binary1248-android'], max_builds=1, properties={'parallel': 1}), Worker('binary1248-coverity', private.slave_passwords['binary1248-coverity'], max_builds=1, properties={'parallel': 1}) ]
def masterConfig(): global path_to_git_bundle from buildbot_travis import TravisConfigurator path_to_git_bundle = os.path.abspath(os.path.join(os.path.dirname(__file__), "test.git.bundle")) with open("sample.yml", "w") as f: f.write(sample_yml % dict(path_to_git_bundle=path_to_git_bundle)) c = {} c['workers'] = [Worker("local1", "p")] TravisConfigurator(c, os.getcwd()).fromYaml("sample.yml") return c
# Note that the *same* configuration objects are used for both runs of the # master. This is a more strenuous test than strictly required, since a master # will generally re-execute master.cfg on startup. However, it's good form and # will help to flush out any bugs that may otherwise be difficult to find. c = BuildmasterConfig = {} from buildbot.changes.filter import ChangeFilter from buildbot.changes.pb import PBChangeSource from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.schedulers.basic import AnyBranchScheduler from buildbot.schedulers.forcesched import ForceScheduler from buildbot.steps.shell import ShellCommand from buildbot.worker import Worker c['workers'] = [Worker("local1", "localpw")] c['protocols'] = {'pb': {'port': 'tcp:0'}} c['change_source'] = [] c['change_source'] = PBChangeSource() c['schedulers'] = [] c['schedulers'].append( AnyBranchScheduler(name="all", change_filter=ChangeFilter(project_re='^testy/'), treeStableTimer=1 * 60, builderNames=[ 'testy', ])) c['schedulers'].append(ForceScheduler(name="force", builderNames=["testy"])) f1 = BuildFactory() f1.addStep(ShellCommand(command='echo hi')) c['builders'] = []
def loadBuilderConfig(c, is_test_mode_enabled=False): # FIXME: These file handles are leaked. if is_test_mode_enabled: passwords = {} else: passwords = json.load(open('passwords.json')) results_server_api_key = passwords.get('results-server-api-key') if results_server_api_key: os.environ['RESULTS_SERVER_API_KEY'] = results_server_api_key config = json.load(open('config.json')) if USE_BUILDBOT_VERSION2: c['workers'] = [Worker(worker['name'], passwords.get(worker['name'], 'password'), max_builds=1) for worker in config['workers']] else: c['slaves'] = [BuildSlave(worker['name'], passwords.get(worker['name'], 'password'), max_builds=1) for worker in config['workers']] c['schedulers'] = [] for scheduler in config['schedulers']: if "change_filter" in scheduler: scheduler["change_filter"] = globals()[scheduler["change_filter"]] schedulerClassName = scheduler.pop('type') schedulerClass = globals()[schedulerClassName] c['schedulers'].append(schedulerClass(**scheduler)) forceScheduler = ForceScheduler( name="force", builderNames=[str(builder['name']) for builder in config['builders']], reason=StringParameter(name="reason", default="", size=40), # Validate SVN revision: number or empty string revision=StringParameter(name="revision", default="", regex=re.compile(r'^(\d*)$')), # Disable default enabled input fields: branch, repository, project, additional properties branch=FixedParameter(name="branch"), repository=FixedParameter(name="repository"), project=FixedParameter(name="project"), properties=[BooleanParameter(name="is_clean", label="Force Clean build")] ) c['schedulers'].append(forceScheduler) c['builders'] = [] for builder in config['builders']: for workerName in builder['workernames']: for worker in config['workers']: if worker['name'] != workerName or worker['platform'] == '*': continue if worker['platform'] != builder['platform']: raise Exception('Builder {} is for platform {} but has worker {} for platform {}!'.format(builder['name'], builder['platform'], worker['name'], worker['platform'])) break if not USE_BUILDBOT_VERSION2: builder['slavenames'] = builder.pop('workernames') platform = builder['platform'] factoryName = builder.pop('factory') factory = globals()[factoryName] factorykwargs = {} for key in "platform", "configuration", "architectures", "triggers", "additionalArguments", "device_model": value = builder.pop(key, None) if value: factorykwargs[key] = value builder["factory"] = factory(**factorykwargs) builder_name = builder['name'] if len(builder_name) > BUILDER_NAME_LENGTH_LIMIT: raise Exception('Builder name "{}" is longer than maximum allowed by Buildbot ({} characters).'.format(builder_name, BUILDER_NAME_LENGTH_LIMIT)) if not buildbot_identifiers_re.match(builder_name): raise Exception('Builder name "{}" is not a valid buildbot identifier.'.format(builder_name)) for step in builder["factory"].steps: if USE_BUILDBOT_VERSION2: step_name = step.buildStep().name else: step_name = step[0].name if len(step_name) > STEP_NAME_LENGTH_LIMIT: raise Exception('step name "{}" is longer than maximum allowed by Buildbot ({} characters).'.format(step_name, STEP_NAME_LENGTH_LIMIT)) if not buildbot_identifiers_re.match(step_name): raise Exception('step name "{}" is not a valid buildbot identifier.'.format(step_name)) if platform.startswith('mac'): category = 'AppleMac' elif platform.startswith('ios'): category = 'iOS' elif platform == 'win': category = 'AppleWin' elif platform.startswith('gtk'): category = 'GTK' elif platform.startswith('wpe'): category = 'WPE' elif platform == 'wincairo': category = 'WinCairo' elif platform.startswith('playstation'): category = 'PlayStation' else: category = 'misc' if (category in ('AppleMac', 'AppleWin', 'iOS')) and factoryName != 'BuildFactory': builder['nextBuild'] = pickLatestBuild if USE_BUILDBOT_VERSION2: builder['tags'] = getTagsForBuilder(builder) else: builder['category'] = category c['builders'].append(builder)
macosx_11_0_workers = ["build-osx-3"] # GitHub OAuth configuration github_oauth_client_id = "c8d9d6b985fdff918e58" github_oauth_client_secret = "89a456962e91303fbb2de4d3a876dcf047d8e60e" # These files contain metadata (incl. passwords) of the workers and users. workers_fn = "workers.json" users_fn = "users.json" # Load the workers from an external JSON file, since we don't want # the worker passwords to become publicly visible. workers = [] for worker_spec in json.load(open(workers_fn, 'r')): name = worker_spec.pop('name') workers.append(Worker(name, **worker_spec)) # Do the same for the user accounts of the web interface. users = [] for user_spec in json.load(open(users_fn, 'r')): users.append((str(user_spec[0]), str(user_spec[1]))) from fnmatch import fnmatchcase def _matches(file, pattern): file = file.lstrip('/') if pattern.startswith('/'): return fnmatchcase(file, pattern[1:]) # Relative pattern. Recurse through the hierarchy to see if it matches.
import simplejson as json from buildbot.worker import Worker ### Fetch workers from json file Workers = [] WorkerTags = {} for worker in json.load(open('../workers.json')): tags = worker.pop('tags', ['default']) name = worker.pop('name') for tag in tags: if tag not in WorkerTags: WorkerTags[tag] = [] WorkerTags[tag].append(name) Workers.append(Worker(name, worker.pop('pass'), **worker))
def loadBuilderConfig(c, is_test_mode_enabled=False, master_prefix_path='./'): # FIXME: These file handles are leaked. config = json.load(open(os.path.join(master_prefix_path, 'config.json'))) if is_test_mode_enabled: passwords = {} else: passwords = json.load( open(os.path.join(master_prefix_path, 'passwords.json'))) results_server_api_key = passwords.get('results-server-api-key') if results_server_api_key: os.environ['RESULTS_SERVER_API_KEY'] = results_server_api_key checkWorkersAndBuildersForConsistency(config, config['workers'], config['builders']) checkValidSchedulers(config, config['schedulers']) c['workers'] = [ Worker(worker['name'], passwords.get(worker['name'], 'password'), max_builds=1) for worker in config['workers'] ] c['schedulers'] = [] for scheduler in config['schedulers']: if "change_filter" in scheduler: scheduler["change_filter"] = globals()[scheduler["change_filter"]] schedulerClassName = scheduler.pop('type') schedulerClass = globals()[schedulerClassName] c['schedulers'].append(schedulerClass(**scheduler)) # Setup force schedulers builderNames = [str(builder['name']) for builder in config['builders']] reason = StringParameter(name='reason', default='', size=40) properties = [BooleanParameter(name='is_clean', label='Force Clean build')] forceScheduler = ForceScheduler(name='force', builderNames=builderNames, reason=reason, properties=properties) c['schedulers'].append(forceScheduler) c['builders'] = [] for builder in config['builders']: builder['tags'] = getTagsForBuilder(builder) platform = builder['platform'] factoryName = builder.pop('factory') factory = globals()[factoryName] factorykwargs = {} for key in [ 'platform', 'configuration', 'architectures', 'triggers', 'additionalArguments', 'device_model' ]: value = builder.pop(key, None) if value: factorykwargs[key] = value builder['factory'] = factory(**factorykwargs) builder_name = builder['name'] for step in builder["factory"].steps: step_name = step.buildStep().name if len(step_name) > STEP_NAME_LENGTH_LIMIT: raise Exception( 'step name "{}" is longer than maximum allowed by Buildbot ({} characters).' .format(step_name, STEP_NAME_LENGTH_LIMIT)) if not buildbot_identifiers.ident_re.match(step_name): raise Exception( 'step name "{}" is not a valid buildbot identifier.'. format(step_name)) if platform.startswith('mac'): category = 'AppleMac' elif platform.startswith('ios'): category = 'iOS' elif platform == 'win': category = 'AppleWin' elif platform.startswith('gtk'): category = 'GTK' elif platform.startswith('wpe'): category = 'WPE' elif platform == 'wincairo': category = 'WinCairo' elif platform.startswith('playstation'): category = 'PlayStation' else: category = 'misc' if (category in ('AppleMac', 'AppleWin', 'iOS')) and factoryName != 'BuildFactory': builder['nextBuild'] = pickLatestBuild c['builders'].append(builder)
"buildHorizon": 50, "logHorizon": 25, # More caching, less IO. "caches": { "Changes": 5000, "Builds": 500, "BuildRequests": 200, "SourceStamps": 200, }, "prioritizeBuilders": prioritize_builders, "workers": [ Worker("windows", BUILDSLAVES_PASSWORDS["windows"], max_builds=1), Worker("osx", BUILDSLAVES_PASSWORDS["osx"], max_builds=1), Worker("debian", BUILDSLAVES_PASSWORDS["debian"], max_builds=1), Worker("ubuntu", BUILDSLAVES_PASSWORDS["ubuntu"], max_builds=1), Worker("freebsd", BUILDSLAVES_PASSWORDS["freebsd"], max_builds=1), Worker("ec2-ubu64-nouveau", BUILDSLAVES_PASSWORDS["ec2-ubu64-nouveau"], keepalive_interval=60, max_builds=1), Worker("ec2-ubu64-nv", BUILDSLAVES_PASSWORDS["ec2-ubu64-nv"], keepalive_interval=60, max_builds=1), Worker("ec2-win64-nv", BUILDSLAVES_PASSWORDS["ec2-win64-nv"], keepalive_interval=60,