def verifyCode(self, code): # everything in deferToThread is not counted with trial --coverage :-( def thd(client_id, client_secret): url = self.tokenUri data = {'redirect_uri': self.loginUri, 'code': code, 'grant_type': self.grantType} auth = None if self.getTokenUseAuthHeaders: auth = (client_id, client_secret) else: data.update( {'client_id': client_id, 'client_secret': client_secret}) data.update(self.tokenUriAdditionalParams) response = requests.post( url, data=data, auth=auth, verify=self.sslVerify) response.raise_for_status() responseContent = bytes2unicode(response.content) try: content = json.loads(responseContent) except ValueError: content = parse_qs(responseContent) for k, v in content.items(): content[k] = v[0] except TypeError: content = responseContent session = self.createSessionFromToken(content) return self.getUserInfoFromOAuthClient(session) p = Properties() p.master = self.master client_id = yield p.render(self.clientId) client_secret = yield p.render(self.clientSecret) result = yield threads.deferToThread(thd, client_id, client_secret) return result
def _downloadSshPrivateKeyIfNeeded(self): if self.sshPrivateKey is None: defer.returnValue(RC_SUCCESS) p = Properties() p.master = self.master private_key = yield p.render(self.sshPrivateKey) host_key = yield p.render(self.sshHostKey) # not using self.workdir because it may be changed depending on step # options workdir = self._getSshDataWorkDir() yield self.runMkdir(self._getSshDataPath()) if not self.supportsSshPrivateKeyAsEnvOption: yield self.downloadFileContentToWorker(self._getSshWrapperScriptPath(), self._getSshWrapperScript(), workdir=workdir, mode=0o700) yield self.downloadFileContentToWorker(self._getSshPrivateKeyPath(), private_key, workdir=workdir, mode=0o400) if self.sshHostKey is not None: known_hosts_contents = getSshKnownHostsContents(host_key) yield self.downloadFileContentToWorker(self._getSshHostKeyPath(), known_hosts_contents, workdir=workdir, mode=0o400) self.didDownloadSshPrivateKey = True defer.returnValue(RC_SUCCESS)
def renderSecrets(self, *args): # Properties import to resolve cyclic import issue from buildbot.process.properties import Properties p = Properties() p.master = self.master if len(args) == 1: return p.render(args[0]) return defer.gatherResults([p.render(s) for s in args], consumeErrors=True)
def _render_docker_properties(self): """Render docker properties dinamically. Docker specific configuration defined in the DockerBuilder instances are passed as properties to the DockerLatentWorker. Only scalar properies are allowed in BuilderConfig instances, so defining docker volumes referring the builddir would require boilerplate and deeper understanding how the build properties are propegated through the buildbot objects. So using traditional property interpolation with properties like the docker image's workdir, builddir, or the buildername makes the volume definition easier. Note that the builddir and workerbuilddir variables of the builder config are different from the ones we see on the buildbot ui under the properties tab. License note: The property descriptions below are copied from the buildbot docs. self.builddir: Specifies the name of a subdirectory of the master’s basedir in which everything related to this builder will be stored. This holds build status information. If not set, this parameter defaults to the builder name, with some characters escaped. Each builder must have a unique build directory. self.workerbuilddir: Specifies the name of a subdirectory (under the worker’s configured base directory) in which everything related to this builder will be placed on the worker. This is where checkouts, compiles, and tests are run. If not set, defaults to builddir. If a worker is connected to multiple builders that share the same workerbuilddir, make sure the worker is set to run one build at a time or ensure this is fine to run multiple builds from the same directory simultaneously. """ props = Properties(buildername=self.name, builddir=self.builddir, workerbuilddir=self.workerbuilddir, docker_image=str(self.image), docker_workdir=self.image.workdir) self.properties.update({ 'docker_image': str(self.image), 'docker_workdir': self.image.workdir, 'docker_volumes': props.render(self.volumes).result, 'docker_hostconfig': props.render(self.hostconfig).result, })
def _downloadSshPrivateKeyIfNeeded(self): if self.sshPrivateKey is None: defer.returnValue(RC_SUCCESS) p = Properties() p.master = self.master private_key = yield p.render(self.sshPrivateKey) # not using self.workdir because it may be changed depending on step # options workdir = self._getSshDataWorkDir() rel_key_path = self.build.path_module.relpath( self._getSshPrivateKeyPath(), workdir) rel_wrapper_script_path = self.build.path_module.relpath( self._getSshWrapperScriptPath(), workdir) yield self.runMkdir(self._getSshDataPath()) if not self.supportsSshPrivateKeyAsEnvOption: yield self.downloadFileContentToWorker(rel_wrapper_script_path, self._getSshWrapperScript(), workdir=workdir, mode=0o700) yield self.downloadFileContentToWorker(rel_key_path, private_key, workdir=workdir, mode=0o400) self.didDownloadSshPrivateKey = True defer.returnValue(RC_SUCCESS)
def canStartBuild(self, workerforbuilder, buildrequest): can_start = True # check whether the locks that the build will acquire can actually be # acquired locks = self.config.locks if IRenderable.providedBy(locks): # collect properties that would be set for a build if we # started it now and render locks using it props = Properties() Build.setupPropertiesKnownBeforeBuildStarts( props, [buildrequest], self, workerforbuilder) locks = yield props.render(locks) locks = [(self.botmaster.getLockFromLockAccess(access), access) for access in locks] if locks: can_start = Build._canAcquireLocks(locks, workerforbuilder) if can_start is False: defer.returnValue(can_start) if callable(self.config.canStartBuild): can_start = yield self.config.canStartBuild( self, workerforbuilder, buildrequest) defer.returnValue(can_start)
def canStartBuild(self, workerforbuilder, buildrequest): can_start = True # check whether the locks that the build will acquire can actually be # acquired locks = self.config.locks if IRenderable.providedBy(locks): # collect properties that would be set for a build if we # started it now and render locks using it props = Properties() Build.setupPropertiesKnownBeforeBuildStarts(props, [buildrequest], self, workerforbuilder) locks = yield props.render(locks) locks = [(self.botmaster.getLockFromLockAccess(access), access) for access in locks] if locks: can_start = Build._canAcquireLocks(locks, workerforbuilder) if can_start is False: defer.returnValue(can_start) if callable(self.config.canStartBuild): can_start = yield self.config.canStartBuild(self, workerforbuilder, buildrequest) defer.returnValue(can_start)
def reconfigServiceWithSibling(self, sibling): # only reconfigure if sibling is configured differently. # sibling == self is using ComparableMixin's implementation # only compare compare_attrs if self.configured and sibling == self: defer.returnValue(None) self.configured = True # render renderables in parallel # Properties import to resolve cyclic import issue from buildbot.process.properties import Properties p = Properties() p.master = self.master # render renderables in parallel secrets = [] kwargs = {} accumulateClassList(self.__class__, 'secrets', secrets) for k, v in sibling._config_kwargs.items(): if k in secrets: value = yield p.render(v) setattr(self, k, value) kwargs.update({k: value}) else: kwargs.update({k: v}) d = yield self.reconfigService(*sibling._config_args, **sibling._config_kwargs) defer.returnValue(d)
def reconfigServiceWithSibling(self, sibling): # only reconfigure if sibling is configured differently. # sibling == self is using ComparableMixin's implementation # only compare compare_attrs if self.configured and sibling == self: return None self.configured = True # render renderables in parallel # Properties import to resolve cyclic import issue from buildbot.process.properties import Properties p = Properties() p.master = self.master # render renderables in parallel secrets = [] kwargs = {} accumulateClassList(self.__class__, 'secrets', secrets) for k, v in sibling._config_kwargs.items(): if k in secrets: # for non reconfigurable services, we force the attribute v = yield p.render(v) setattr(sibling, k, v) setattr(self, k, v) kwargs[k] = v d = yield self.reconfigService(*sibling._config_args, **kwargs) return d
def _get_pr_files(self, repo, number): """ Get Files that belong to the Pull Request :param repo: the repo full name, ``{owner}/{project}``. e.g. ``buildbot/buildbot`` :param number: the pull request number. """ headers = {"User-Agent": "Buildbot"} if self._token: p = Properties() p.master = self.master token = yield p.render(self._token) headers["Authorization"] = "token " + token url = f"/repos/{repo}/pulls/{number}/files" http = yield httpclientservice.HTTPClientService.getService( self.master, self.github_api_endpoint, headers=headers, debug=self.debug, verify=self.verify, ) res = yield http.get(url) if 200 <= res.code < 300: data = yield res.json() return [f["filename"] for f in data] log.msg(f'Failed fetching PR files: response code {res.code}') return []
def canStartWithWorkerForBuilder(self, workerforbuilder, buildrequests=None): locks = self.config.locks if IRenderable.providedBy(locks): if buildrequests is None: raise RuntimeError("buildrequests parameter must be specified " " when using renderable builder locks. Not " "specifying buildrequests is deprecated") # collect properties that would be set for a build if we # started it now and render locks using it props = Properties() Build.setupPropertiesKnownBeforeBuildStarts(props, buildrequests, self, workerforbuilder) locks = yield props.render(locks) # Make sure we don't warn and throw an exception at the same time if buildrequests is None: warnings.warn( "Not passing corresponding buildrequests to " "Builder.canStartWithWorkerForBuilder is deprecated") locks = [(self.botmaster.getLockFromLockAccess(access), access) for access in locks] can_start = Build.canStartWithWorkerForBuilder(locks, workerforbuilder) defer.returnValue(can_start)
def _get_commit_msg(self, repo, sha): ''' :param repo: the repo full name, ``{owner}/{project}``. e.g. ``buildbot/buildbot`` ''' headers = { 'User-Agent': 'Buildbot', } if self._token: p = Properties() p.master = self.master token = yield p.render(self._token) headers['Authorization'] = 'token ' + token url = f'/repos/{repo}/commits/{sha}' http = yield httpclientservice.HTTPClientService.getService( self.master, self.github_api_endpoint, headers=headers, debug=self.debug, verify=self.verify) res = yield http.get(url) if 200 <= res.code < 300: data = yield res.json() return data['commit']['message'] log.msg(f'Failed fetching PR commit message: response code {res.code}') return 'No message field'
def canStartWithWorkerForBuilder(self, workerforbuilder, buildrequests=None): locks = self.config.locks if IRenderable.providedBy(locks): if buildrequests is None: raise RuntimeError("buildrequests parameter must be specified " " when using renderable builder locks. Not " "specifying buildrequests is deprecated") # collect properties that would be set for a build if we # started it now and render locks using it props = Properties() Build.setupPropertiesKnownBeforeBuildStarts( props, buildrequests, self, workerforbuilder) locks = yield props.render(locks) # Make sure we don't warn and throw an exception at the same time if buildrequests is None: warnings.warn("Not passing corresponding buildrequests to " "Builder.canStartWithWorkerForBuilder is deprecated") locks = [(self.botmaster.getLockFromLockAccess(access), access) for access in locks] can_start = Build.canStartWithWorkerForBuilder(locks, workerforbuilder) defer.returnValue(can_start)
def reconfigServiceWithSibling(self, sibling): # only reconfigure if sibling is configured differently. # sibling == self is using ComparableMixin's implementation # only compare compare_attrs if self.configured and sibling == self: defer.returnValue(None) self.configured = True # render renderables in parallel # Properties import to resolve cyclic import issue from buildbot.process.properties import Properties p = Properties() p.master = self.master # render renderables in parallel secrets = [] kwargs = {} accumulateClassList(self.__class__, 'secrets', secrets) for k, v in sibling._config_kwargs.items(): if k in secrets: # for non reconfigurable services, we force the attribute v = yield p.render(v) setattr(sibling, k, v) setattr(self, k, v) kwargs[k] = v d = yield self.reconfigService(*sibling._config_args, **kwargs) defer.returnValue(d)
def _downloadSshPrivateKeyIfNeeded(self): if self.sshPrivateKey is None: return RC_SUCCESS p = Properties() p.master = self.master private_key = yield p.render(self.sshPrivateKey) host_key = yield p.render(self.sshHostKey) known_hosts_contents = yield p.render(self.sshKnownHosts) # not using self.workdir because it may be changed depending on step # options workdir = self._getSshDataWorkDir() ssh_data_path = self._getSshDataPath() yield self.runMkdir(ssh_data_path) private_key_path = self._getSshPrivateKeyPath(ssh_data_path) yield self.downloadFileContentToWorker(private_key_path, private_key, workdir=workdir, mode=0o400) known_hosts_path = None if self.sshHostKey is not None or self.sshKnownHosts is not None: known_hosts_path = self._getSshHostKeyPath(ssh_data_path) if self.sshHostKey is not None: known_hosts_contents = getSshKnownHostsContents(host_key) yield self.downloadFileContentToWorker(known_hosts_path, known_hosts_contents, workdir=workdir, mode=0o400) if not self.supportsSshPrivateKeyAsEnvOption: script_path = self._getSshWrapperScriptPath(ssh_data_path) script_contents = getSshWrapperScriptContents( private_key_path, known_hosts_path) yield self.downloadFileContentToWorker(script_path, script_contents, workdir=workdir, mode=0o700) self.didDownloadSshPrivateKey = True return RC_SUCCESS
def getChanges(self, request): """ Reponds only to POST events and starts the build process :arguments: request the http request object """ expected_secret = isinstance(self.options, dict) and self.options.get('secret') if expected_secret: received_secret = request.getHeader(_HEADER_GITLAB_TOKEN) received_secret = bytes2unicode(received_secret) p = Properties() p.master = self.master expected_secret_value = yield p.render(expected_secret) if received_secret != expected_secret_value: raise ValueError("Invalid secret") try: content = request.content.read() payload = json.loads(bytes2unicode(content)) except Exception as e: raise ValueError("Error loading JSON: " + str(e)) from e event_type = request.getHeader(_HEADER_EVENT) event_type = bytes2unicode(event_type) # newer version of gitlab have a object_kind parameter, # which allows not to use the http header event_type = payload.get('object_kind', event_type) codebase = request.args.get(b'codebase', [None])[0] codebase = bytes2unicode(codebase) if event_type in ("push", "tag_push", "Push Hook"): user = payload['user_name'] repo = payload['repository']['name'] repo_url = payload['repository']['url'] changes = self._process_change(payload, user, repo, repo_url, event_type, codebase=codebase) elif event_type == 'merge_request': changes = self._process_merge_request_change(payload, event_type, codebase=codebase) else: changes = [] if changes: log.msg( f"Received {len(changes)} changes from {event_type} gitlab event" ) return (changes, 'git')
def _get_payload(self, request): content = request.content.read() content = bytes2unicode(content) signature = request.getHeader(_HEADER_SIGNATURE) signature = bytes2unicode(signature) if not signature and self._strict: raise ValueError('Request has no required signature') if self._secret and signature: try: hash_type, hexdigest = signature.split('=') except ValueError: raise ValueError( 'Wrong signature format: {}'.format(signature)) if hash_type != 'sha1': raise ValueError('Unknown hash type: {}'.format(hash_type)) p = Properties() p.master = self.master rendered_secret = yield p.render(self._secret) mac = hmac.new(unicode2bytes(rendered_secret), msg=unicode2bytes(content), digestmod=sha1) def _cmp(a, b): try: # try the more secure compare_digest() first from hmac import compare_digest return compare_digest(a, b) except ImportError: # pragma: no cover # and fallback to the insecure simple comparison otherwise return a == b if not _cmp(bytes2unicode(mac.hexdigest()), hexdigest): raise ValueError('Hash mismatch') content_type = request.getHeader(b'Content-Type') if content_type == b'application/json': payload = json.loads(content) elif content_type == b'application/x-www-form-urlencoded': payload = json.loads(bytes2unicode(request.args[b'payload'][0])) else: raise ValueError('Unknown content type: {}'.format(content_type)) log.msg("Payload: {}".format(payload), logLevel=logging.DEBUG) return payload
def _downloadSshPrivateKeyIfNeeded(self): if self.sshPrivateKey is None: return RC_SUCCESS p = Properties() p.master = self.master private_key = yield p.render(self.sshPrivateKey) host_key = yield p.render(self.sshHostKey) # not using self.workdir because it may be changed depending on step # options workdir = self._getSshDataWorkDir() ssh_data_path = self._getSshDataPath() yield self.runMkdir(ssh_data_path) if not self.supportsSshPrivateKeyAsEnvOption: script_path = self._getSshWrapperScriptPath(ssh_data_path) script_contents = getSshWrapperScriptContents( self._getSshPrivateKeyPath(ssh_data_path)) yield self.downloadFileContentToWorker(script_path, script_contents, workdir=workdir, mode=0o700) private_key_path = self._getSshPrivateKeyPath(ssh_data_path) yield self.downloadFileContentToWorker(private_key_path, private_key, workdir=workdir, mode=0o400) if self.sshHostKey is not None: known_hosts_path = self._getSshHostKeyPath(ssh_data_path) known_hosts_contents = getSshKnownHostsContents(host_key) yield self.downloadFileContentToWorker(known_hosts_path, known_hosts_contents, workdir=workdir, mode=0o400) self.didDownloadSshPrivateKey = True return RC_SUCCESS
def getLoginURL(self, redirect_url): """ Returns the url to redirect the user to for user consent """ p = Properties() p.master = self.master clientId = yield p.render(self.clientId) oauth_params = {'redirect_uri': self.loginUri, 'client_id': clientId, 'response_type': 'code'} if redirect_url is not None: oauth_params['state'] = urlencode(dict(redirect=redirect_url)) oauth_params.update(self.authUriAdditionalParams) sorted_oauth_params = sorted(oauth_params.items(), key=lambda val: val[0]) return "%s?%s" % (self.authUri, urlencode(sorted_oauth_params))
def newChangeSource(self, owner, repo, endpoint='https://api.github.com', **kwargs): http_headers = {'User-Agent': 'Buildbot'} token = kwargs.get('token', None) if token: p = Properties() p.master = self.master token = yield p.render(token) http_headers.update({'Authorization': 'token ' + token}) self._http = yield fakehttpclientservice.HTTPClientService.getService( self.master, self, endpoint, headers=http_headers) self.changesource = GitHubPullrequestPoller(owner, repo, **kwargs)
def _get_payload(self, request): content = request.content.read() content = bytes2unicode(content) signature = request.getHeader(_HEADER_SIGNATURE) signature = bytes2unicode(signature) if not signature and self._strict: raise ValueError('Request has no required signature') if self._secret and signature: try: hash_type, hexdigest = signature.split('=') except ValueError as e: raise ValueError(f'Wrong signature format: {signature}') from e if hash_type != 'sha1': raise ValueError(f'Unknown hash type: {hash_type}') p = Properties() p.master = self.master rendered_secret = yield p.render(self._secret) mac = hmac.new(unicode2bytes(rendered_secret), msg=unicode2bytes(content), digestmod=sha1) def _cmp(a, b): return hmac.compare_digest(a, b) if not _cmp(bytes2unicode(mac.hexdigest()), hexdigest): raise ValueError('Hash mismatch') content_type = request.getHeader(b'Content-Type') if content_type == b'application/json': payload = json.loads(content) elif content_type == b'application/x-www-form-urlencoded': payload = json.loads(bytes2unicode(request.args[b'payload'][0])) else: raise ValueError(f'Unknown content type: {content_type}') log.msg(f"Payload: {payload}", logLevel=logging.DEBUG) return payload
def requestAvatarId(self, creds): p = Properties() p.master = self.master username = bytes2unicode(creds.username) try: yield self.master.initLock.acquire() if username in self.users: password, _ = self.users[username] password = yield p.render(password) matched = yield defer.maybeDeferred(creds.checkPassword, unicode2bytes(password)) if not matched: log.msg("invalid login from user '{}'".format(username)) raise error.UnauthorizedLogin() defer.returnValue(creds.username) log.msg("invalid login from unknown user '{}'".format(username)) raise error.UnauthorizedLogin() finally: eventually(self.master.initLock.release)
def requestAvatarId(self, creds): p = Properties() p.master = self.master username = bytes2unicode(creds.username) try: yield self.master.initLock.acquire() if username in self.users: password, _ = self.users[username] password = yield p.render(password) matched = yield defer.maybeDeferred( creds.checkPassword, unicode2bytes(password)) if not matched: log.msg("invalid login from user '%s'" % creds.username) raise error.UnauthorizedLogin() defer.returnValue(creds.username) log.msg("invalid login from unknown user '%s'" % creds.username) raise error.UnauthorizedLogin() finally: yield self.master.initLock.release()
def requestAvatarId(self, creds): p = Properties() p.master = self.master username = bytes2unicode(creds.username) try: yield self.master.initLock.acquire() if username in self.users: password, _ = self.users[username] password = yield p.render(password) matched = creds.checkPassword(unicode2bytes(password)) if not matched: log.msg("invalid login from user '{}'".format(username)) raise error.UnauthorizedLogin() return creds.username log.msg("invalid login from unknown user '{}'".format(username)) raise error.UnauthorizedLogin() finally: # brake the callback stack by returning to the reactor # before waking up other waiters eventually(self.master.initLock.release)
def requestAvatarId(self, creds): p = Properties() p.master = self.master username = bytes2unicode(creds.username) try: yield self.master.initLock.acquire() if username in self.users: password, _ = self.users[username] password = yield p.render(password) matched = yield defer.maybeDeferred( creds.checkPassword, unicode2bytes(password)) if not matched: log.msg("invalid login from user '{}'".format(username)) raise error.UnauthorizedLogin() return creds.username log.msg("invalid login from unknown user '{}'".format(username)) raise error.UnauthorizedLogin() finally: # brake the callback stack by returning to the reactor # before waking up other waiters eventually(self.master.initLock.release)
def getChanges(self, request): secret = None if isinstance(self.options, dict): secret = self.options.get('secret') try: content = request.content.read() content_text = bytes2unicode(content) payload = json.loads(content_text) except Exception as exception: raise ValueError('Error loading JSON: ' + str(exception)) if secret is not None: p = Properties() p.master = self.master rendered_secret = yield p.render(secret) signature = hmac.new(unicode2bytes(rendered_secret), unicode2bytes(content_text.strip()), digestmod=hashlib.sha256) header_signature = bytes2unicode( request.getHeader(_HEADER_SIGNATURE)) if signature.hexdigest() != header_signature: raise ValueError('Invalid secret') event_type = bytes2unicode(request.getHeader(_HEADER_EVENT_TYPE)) log.msg("Received event '{}' from gitea".format(event_type)) codebases = request.args.get('codebase', [None]) codebase = bytes2unicode(codebases[0]) changes = [] handler_function = getattr(self, 'process_{}'.format(event_type), None) if not handler_function: log.msg("Ignoring gitea event '{}'".format(event_type)) else: changes = handler_function(payload, event_type, codebase) return (changes, 'git')
class TestWithProperties(unittest.TestCase): def setUp(self): self.props = Properties() def testBasic(self): # test basic substitution with WithProperties self.props.setProperty("revision", "47", "test") command = WithProperties("build-%s.tar.gz", "revision") self.failUnlessEqual(self.props.render(command), "build-47.tar.gz") def testDict(self): # test dict-style substitution with WithProperties self.props.setProperty("other", "foo", "test") command = WithProperties("build-%(other)s.tar.gz") self.failUnlessEqual(self.props.render(command), "build-foo.tar.gz") def testDictColonMinus(self): # test dict-style substitution with WithProperties self.props.setProperty("prop1", "foo", "test") command = WithProperties("build-%(prop1:-empty)s-%(prop2:-empty)s.tar.gz") self.failUnlessEqual(self.props.render(command), "build-foo-empty.tar.gz") def testDictColonPlus(self): # test dict-style substitution with WithProperties self.props.setProperty("prop1", "foo", "test") command = WithProperties("build-%(prop1:+exists)s-%(prop2:+exists)s.tar.gz") self.failUnlessEqual(self.props.render(command), "build-exists-.tar.gz") def testEmpty(self): # None should render as '' self.props.setProperty("empty", None, "test") command = WithProperties("build-%(empty)s.tar.gz") self.failUnlessEqual(self.props.render(command), "build-.tar.gz") def testRecursiveList(self): self.props.setProperty("x", 10, "test") self.props.setProperty("y", 20, "test") command = [ WithProperties("%(x)s %(y)s"), "and", WithProperties("%(y)s %(x)s") ] self.failUnlessEqual(self.props.render(command), ["10 20", "and", "20 10"]) def testRecursiveTuple(self): self.props.setProperty("x", 10, "test") self.props.setProperty("y", 20, "test") command = ( WithProperties("%(x)s %(y)s"), "and", WithProperties("%(y)s %(x)s") ) self.failUnlessEqual(self.props.render(command), ("10 20", "and", "20 10")) def testRecursiveDict(self): self.props.setProperty("x", 10, "test") self.props.setProperty("y", 20, "test") command = { WithProperties("%(x)s %(y)s") : WithProperties("%(y)s %(x)s") } self.failUnlessEqual(self.props.render(command), {"10 20" : "20 10"}) def testLambdaSubst(self): command = WithProperties('%(foo)s', foo=lambda _: 'bar') self.failUnlessEqual(self.props.render(command), 'bar') def testLambdaOverride(self): self.props.setProperty('x', 10, 'test') command = WithProperties('%(x)s', x=lambda _: 20) self.failUnlessEqual(self.props.render(command), '20') def testLambdaCallable(self): self.assertRaises(ValueError, lambda: WithProperties('%(foo)s', foo='bar')) def testLambdaUseExisting(self): self.props.setProperty('x', 10, 'test') self.props.setProperty('y', 20, 'test') command = WithProperties('%(z)s', z=lambda pmap: pmap['x'] + pmap['y']) self.failUnlessEqual(self.props.render(command), '30')
class TestWithProperties(unittest.TestCase): def setUp(self): self.props = Properties() def testBasic(self): # test basic substitution with WithProperties self.props.setProperty("revision", "47", "test") command = WithProperties("build-%s.tar.gz", "revision") self.failUnlessEqual(self.props.render(command), "build-47.tar.gz") def testDict(self): # test dict-style substitution with WithProperties self.props.setProperty("other", "foo", "test") command = WithProperties("build-%(other)s.tar.gz") self.failUnlessEqual(self.props.render(command), "build-foo.tar.gz") def testDictColonMinus(self): # test dict-style substitution with WithProperties self.props.setProperty("prop1", "foo", "test") command = WithProperties("build-%(prop1:-empty)s-%(prop2:-empty)s.tar.gz") self.failUnlessEqual(self.props.render(command), "build-foo-empty.tar.gz") def testDictColonPlus(self): # test dict-style substitution with WithProperties self.props.setProperty("prop1", "foo", "test") command = WithProperties("build-%(prop1:+exists)s-%(prop2:+exists)s.tar.gz") self.failUnlessEqual(self.props.render(command), "build-exists-.tar.gz") def testEmpty(self): # None should render as '' self.props.setProperty("empty", None, "test") command = WithProperties("build-%(empty)s.tar.gz") self.failUnlessEqual(self.props.render(command), "build-.tar.gz") def testRecursiveList(self): self.props.setProperty("x", 10, "test") self.props.setProperty("y", 20, "test") command = [ WithProperties("%(x)s %(y)s"), "and", WithProperties("%(y)s %(x)s") ] self.failUnlessEqual(self.props.render(command), ["10 20", "and", "20 10"]) def testRecursiveTuple(self): self.props.setProperty("x", 10, "test") self.props.setProperty("y", 20, "test") command = ( WithProperties("%(x)s %(y)s"), "and", WithProperties("%(y)s %(x)s") ) self.failUnlessEqual(self.props.render(command), ("10 20", "and", "20 10")) def testRecursiveDict(self): self.props.setProperty("x", 10, "test") self.props.setProperty("y", 20, "test") command = { WithProperties("%(x)s %(y)s") : WithProperties("%(y)s %(x)s") } self.failUnlessEqual(self.props.render(command), {"10 20" : "20 10"})
class TestProperty(unittest.TestCase): def setUp(self): self.props = Properties() def testIntProperty(self): self.props.setProperty("do-tests", 1, "scheduler") value = Property("do-tests") self.failUnlessEqual(self.props.render(value), 1) def testStringProperty(self): self.props.setProperty("do-tests", "string", "scheduler") value = Property("do-tests") self.failUnlessEqual(self.props.render(value), "string") def testMissingProperty(self): value = Property("do-tests") self.failUnlessEqual(self.props.render(value), None) def testDefaultValue(self): value = Property("do-tests", default="Hello!") self.failUnlessEqual(self.props.render(value), "Hello!") def testIgnoreDefaultValue(self): self.props.setProperty("do-tests", "string", "scheduler") value = Property("do-tests", default="Hello!") self.failUnlessEqual(self.props.render(value), "string") def testIgnoreFalseValue(self): self.props.setProperty("do-tests-string", "", "scheduler") self.props.setProperty("do-tests-int", 0, "scheduler") self.props.setProperty("do-tests-list", [], "scheduler") self.props.setProperty("do-tests-None", None, "scheduler") value = [ Property("do-tests-string", default="Hello!"), Property("do-tests-int", default="Hello!"), Property("do-tests-list", default="Hello!"), Property("do-tests-None", default="Hello!") ] self.failUnlessEqual(self.props.render(value), ["Hello!"] * 4) def testDefaultWhenFalse(self): self.props.setProperty("do-tests-string", "", "scheduler") self.props.setProperty("do-tests-int", 0, "scheduler") self.props.setProperty("do-tests-list", [], "scheduler") self.props.setProperty("do-tests-None", None, "scheduler") value = [ Property("do-tests-string", default="Hello!", defaultWhenFalse=False), Property("do-tests-int", default="Hello!", defaultWhenFalse=False), Property("do-tests-list", default="Hello!", defaultWhenFalse=False), Property("do-tests-None", default="Hello!", defaultWhenFalse=False) ] self.failUnlessEqual(self.props.render(value), ["", 0, [], None])
class TestProperties(unittest.TestCase): def setUp(self): self.props = Properties() def testDictBehavior(self): # note that dictionary-like behavior is deprecated and not exposed to # users! self.props.setProperty("do-tests", 1, "scheduler") self.props.setProperty("do-install", 2, "scheduler") self.assert_(self.props.has_key('do-tests')) self.failUnlessEqual(self.props['do-tests'], 1) self.failUnlessEqual(self.props['do-install'], 2) self.assertRaises(KeyError, lambda: self.props['do-nothing']) self.failUnlessEqual(self.props.getProperty('do-install'), 2) self.assertIn('do-tests', self.props) self.assertNotIn('missing-do-tests', self.props) def testAsList(self): self.props.setProperty("happiness", 7, "builder") self.props.setProperty("flames", True, "tester") self.assertEqual(sorted(self.props.asList()), [('flames', True, 'tester'), ('happiness', 7, 'builder')]) def testAsDict(self): self.props.setProperty("msi_filename", "product.msi", 'packager') self.props.setProperty("dmg_filename", "product.dmg", 'packager') self.assertEqual( self.props.asDict(), dict(msi_filename=('product.msi', 'packager'), dmg_filename=('product.dmg', 'packager'))) def testUpdate(self): self.props.setProperty("x", 24, "old") newprops = {'a': 1, 'b': 2} self.props.update(newprops, "new") self.failUnlessEqual(self.props.getProperty('x'), 24) self.failUnlessEqual(self.props.getPropertySource('x'), 'old') self.failUnlessEqual(self.props.getProperty('a'), 1) self.failUnlessEqual(self.props.getPropertySource('a'), 'new') def testUpdateRuntime(self): self.props.setProperty("x", 24, "old") newprops = {'a': 1, 'b': 2} self.props.update(newprops, "new", runtime=True) self.failUnlessEqual(self.props.getProperty('x'), 24) self.failUnlessEqual(self.props.getPropertySource('x'), 'old') self.failUnlessEqual(self.props.getProperty('a'), 1) self.failUnlessEqual(self.props.getPropertySource('a'), 'new') self.assertEqual(self.props.runtime, set(['a', 'b'])) def testUpdateFromProperties(self): self.props.setProperty("a", 94, "old") self.props.setProperty("x", 24, "old") newprops = Properties() newprops.setProperty('a', 1, "new") newprops.setProperty('b', 2, "new") self.props.updateFromProperties(newprops) self.failUnlessEqual(self.props.getProperty('x'), 24) self.failUnlessEqual(self.props.getPropertySource('x'), 'old') self.failUnlessEqual(self.props.getProperty('a'), 1) self.failUnlessEqual(self.props.getPropertySource('a'), 'new') def testUpdateFromPropertiesNoRuntime(self): self.props.setProperty("a", 94, "old") self.props.setProperty("b", 84, "old") self.props.setProperty("x", 24, "old") newprops = Properties() newprops.setProperty('a', 1, "new", runtime=True) newprops.setProperty('b', 2, "new", runtime=False) newprops.setProperty('c', 3, "new", runtime=True) newprops.setProperty('d', 3, "new", runtime=False) self.props.updateFromPropertiesNoRuntime(newprops) self.failUnlessEqual(self.props.getProperty('a'), 94) self.failUnlessEqual(self.props.getPropertySource('a'), 'old') self.failUnlessEqual(self.props.getProperty('b'), 2) self.failUnlessEqual(self.props.getPropertySource('b'), 'new') self.failUnlessEqual(self.props.getProperty('c'), None) # not updated self.failUnlessEqual(self.props.getProperty('d'), 3) self.failUnlessEqual(self.props.getPropertySource('d'), 'new') self.failUnlessEqual(self.props.getProperty('x'), 24) self.failUnlessEqual(self.props.getPropertySource('x'), 'old') # IProperties methods def test_getProperty(self): self.props.properties['p1'] = (['p', 1], 'test') self.assertEqual(self.props.getProperty('p1'), ['p', 1]) def test_getProperty_default_None(self): self.assertEqual(self.props.getProperty('p1'), None) def test_getProperty_default(self): self.assertEqual(self.props.getProperty('p1', 2), 2) def test_hasProperty_false(self): self.assertFalse(self.props.hasProperty('x')) def test_hasProperty_true(self): self.props.properties['x'] = (False, 'test') self.assertTrue(self.props.hasProperty('x')) def test_has_key_false(self): self.assertFalse(self.props.has_key('x')) def test_setProperty(self): self.props.setProperty('x', 'y', 'test') self.assertEqual(self.props.properties['x'], ('y', 'test')) self.assertNotIn('x', self.props.runtime) def test_setProperty_runtime(self): self.props.setProperty('x', 'y', 'test', runtime=True) self.assertEqual(self.props.properties['x'], ('y', 'test')) self.assertIn('x', self.props.runtime) def test_setProperty_no_source(self): self.assertRaises(TypeError, lambda: self.props.setProperty('x', 'y')) def test_getProperties(self): self.assertIdentical(self.props.getProperties(), self.props) def test_getBuild(self): self.assertIdentical(self.props.getBuild(), self.props.build) def test_render(self): class FakeRenderable(object): implements(IRenderable) def getRenderingFor(self, props): return props.getProperty('x') + 'z' self.props.setProperty('x', 'y', 'test') self.assertEqual(self.props.render(FakeRenderable()), 'yz')
def _render_properties(self): props = Properties(buildername=self.name, builddir=str(self.builddir), workerbuilddir=str(self.workerbuilddir)) rendered = props.render(self.properties) return rendered.result
class TestWithProperties(unittest.TestCase): def setUp(self): self.props = Properties() def testBasic(self): # test basic substitution with WithProperties self.props.setProperty("revision", "47", "test") command = WithProperties("build-%s.tar.gz", "revision") self.failUnlessEqual(self.props.render(command), "build-47.tar.gz") def testDict(self): # test dict-style substitution with WithProperties self.props.setProperty("other", "foo", "test") command = WithProperties("build-%(other)s.tar.gz") self.failUnlessEqual(self.props.render(command), "build-foo.tar.gz") def testDictColonMinus(self): # test dict-style substitution with WithProperties self.props.setProperty("prop1", "foo", "test") command = WithProperties( "build-%(prop1:-empty)s-%(prop2:-empty)s.tar.gz") self.failUnlessEqual(self.props.render(command), "build-foo-empty.tar.gz") def testDictColonPlus(self): # test dict-style substitution with WithProperties self.props.setProperty("prop1", "foo", "test") command = WithProperties( "build-%(prop1:+exists)s-%(prop2:+exists)s.tar.gz") self.failUnlessEqual(self.props.render(command), "build-exists-.tar.gz") def testEmpty(self): # None should render as '' self.props.setProperty("empty", None, "test") command = WithProperties("build-%(empty)s.tar.gz") self.failUnlessEqual(self.props.render(command), "build-.tar.gz") def testRecursiveList(self): self.props.setProperty("x", 10, "test") self.props.setProperty("y", 20, "test") command = [ WithProperties("%(x)s %(y)s"), "and", WithProperties("%(y)s %(x)s") ] self.failUnlessEqual(self.props.render(command), ["10 20", "and", "20 10"]) def testRecursiveTuple(self): self.props.setProperty("x", 10, "test") self.props.setProperty("y", 20, "test") command = (WithProperties("%(x)s %(y)s"), "and", WithProperties("%(y)s %(x)s")) self.failUnlessEqual(self.props.render(command), ("10 20", "and", "20 10")) def testRecursiveDict(self): self.props.setProperty("x", 10, "test") self.props.setProperty("y", 20, "test") command = { WithProperties("%(x)s %(y)s"): WithProperties("%(y)s %(x)s") } self.failUnlessEqual(self.props.render(command), {"10 20": "20 10"})
class TestProperties(unittest.TestCase): def setUp(self): self.props = Properties() def testDictBehavior(self): # note that dictionary-like behavior is deprecated and not exposed to # users! self.props.setProperty("do-tests", 1, "scheduler") self.props.setProperty("do-install", 2, "scheduler") self.assert_(self.props.has_key('do-tests')) self.failUnlessEqual(self.props['do-tests'], 1) self.failUnlessEqual(self.props['do-install'], 2) self.assertRaises(KeyError, lambda : self.props['do-nothing']) self.failUnlessEqual(self.props.getProperty('do-install'), 2) self.assertIn('do-tests', self.props) self.assertNotIn('missing-do-tests', self.props) def testAsList(self): self.props.setProperty("happiness", 7, "builder") self.props.setProperty("flames", True, "tester") self.assertEqual(sorted(self.props.asList()), [ ('flames', True, 'tester'), ('happiness', 7, 'builder') ]) def testAsDict(self): self.props.setProperty("msi_filename", "product.msi", 'packager') self.props.setProperty("dmg_filename", "product.dmg", 'packager') self.assertEqual(self.props.asDict(), dict(msi_filename=('product.msi', 'packager'), dmg_filename=('product.dmg', 'packager'))) def testUpdate(self): self.props.setProperty("x", 24, "old") newprops = { 'a' : 1, 'b' : 2 } self.props.update(newprops, "new") self.failUnlessEqual(self.props.getProperty('x'), 24) self.failUnlessEqual(self.props.getPropertySource('x'), 'old') self.failUnlessEqual(self.props.getProperty('a'), 1) self.failUnlessEqual(self.props.getPropertySource('a'), 'new') def testUpdateRuntime(self): self.props.setProperty("x", 24, "old") newprops = { 'a' : 1, 'b' : 2 } self.props.update(newprops, "new", runtime=True) self.failUnlessEqual(self.props.getProperty('x'), 24) self.failUnlessEqual(self.props.getPropertySource('x'), 'old') self.failUnlessEqual(self.props.getProperty('a'), 1) self.failUnlessEqual(self.props.getPropertySource('a'), 'new') self.assertEqual(self.props.runtime, set(['a', 'b'])) def testUpdateFromProperties(self): self.props.setProperty("a", 94, "old") self.props.setProperty("x", 24, "old") newprops = Properties() newprops.setProperty('a', 1, "new") newprops.setProperty('b', 2, "new") self.props.updateFromProperties(newprops) self.failUnlessEqual(self.props.getProperty('x'), 24) self.failUnlessEqual(self.props.getPropertySource('x'), 'old') self.failUnlessEqual(self.props.getProperty('a'), 1) self.failUnlessEqual(self.props.getPropertySource('a'), 'new') def testUpdateFromPropertiesNoRuntime(self): self.props.setProperty("a", 94, "old") self.props.setProperty("b", 84, "old") self.props.setProperty("x", 24, "old") newprops = Properties() newprops.setProperty('a', 1, "new", runtime=True) newprops.setProperty('b', 2, "new", runtime=False) newprops.setProperty('c', 3, "new", runtime=True) newprops.setProperty('d', 3, "new", runtime=False) self.props.updateFromPropertiesNoRuntime(newprops) self.failUnlessEqual(self.props.getProperty('a'), 94) self.failUnlessEqual(self.props.getPropertySource('a'), 'old') self.failUnlessEqual(self.props.getProperty('b'), 2) self.failUnlessEqual(self.props.getPropertySource('b'), 'new') self.failUnlessEqual(self.props.getProperty('c'), None) # not updated self.failUnlessEqual(self.props.getProperty('d'), 3) self.failUnlessEqual(self.props.getPropertySource('d'), 'new') self.failUnlessEqual(self.props.getProperty('x'), 24) self.failUnlessEqual(self.props.getPropertySource('x'), 'old') # IProperties methods def test_getProperty(self): self.props.properties['p1'] = (['p', 1], 'test') self.assertEqual(self.props.getProperty('p1'), ['p', 1]) def test_getProperty_default_None(self): self.assertEqual(self.props.getProperty('p1'), None) def test_getProperty_default(self): self.assertEqual(self.props.getProperty('p1', 2), 2) def test_hasProperty_false(self): self.assertFalse(self.props.hasProperty('x')) def test_hasProperty_true(self): self.props.properties['x'] = (False, 'test') self.assertTrue(self.props.hasProperty('x')) def test_has_key_false(self): self.assertFalse(self.props.has_key('x')) def test_setProperty(self): self.props.setProperty('x', 'y', 'test') self.assertEqual(self.props.properties['x'], ('y', 'test')) self.assertNotIn('x', self.props.runtime) def test_setProperty_runtime(self): self.props.setProperty('x', 'y', 'test', runtime=True) self.assertEqual(self.props.properties['x'], ('y', 'test')) self.assertIn('x', self.props.runtime) def test_setProperty_no_source(self): self.assertRaises(TypeError, lambda : self.props.setProperty('x', 'y')) def test_getProperties(self): self.assertIdentical(self.props.getProperties(), self.props) def test_getBuild(self): self.assertIdentical(self.props.getBuild(), self.props.build) def test_render(self): class FakeRenderable(object): implements(IRenderable) def getRenderingFor(self, props): return props.getProperty('x') + 'z' self.props.setProperty('x', 'y', 'test') self.assertEqual(self.props.render(FakeRenderable()), 'yz')