def _dovccmdImpl(self, command, args, path, ssh_workdir): full_args = [] full_env = os.environ.copy() if self._isSshPrivateKeyNeededForCommand(command): key_path = self._getSshPrivateKeyPath(ssh_workdir) self._downloadSshPrivateKey(key_path) known_hosts_path = None if self.sshHostKey is not None or self.sshKnownHosts is not None: known_hosts_path = self._getSshKnownHostsPath(ssh_workdir) self._downloadSshKnownHosts(known_hosts_path) self.adjustCommandParamsForSshPrivateKey(full_args, full_env, key_path, None, known_hosts_path) full_args += [command] + args res = yield utils.getProcessOutputAndValue(self.gitbin, full_args, path=path, env=full_env) (stdout, stderr, code) = res stdout = bytes2unicode(stdout, self.encoding) stderr = bytes2unicode(stderr, self.encoding) if code != 0: if code == 128: raise GitError('command {} in {} on repourl {} failed with exit code {}: {}'.format( full_args, path, self.repourl, code, stderr)) raise EnvironmentError('command {} in {} on repourl {} failed with exit code {}: {}'.format( full_args, path, self.repourl, code, stderr)) return stdout.strip()
def perspective_try(self, branch, revision, patch, repository, project, builderNames, who="", comment="", properties=None): log.msg("user %s requesting build on builders %s" % (self.username, builderNames)) if properties is None: properties = {} # build the intersection of the request and our configured list builderNames = self.scheduler.filterBuilderList(builderNames) if not builderNames: return reason = "'try' job" if who: reason += " by user {}".format(bytes2unicode(who)) if comment: reason += " ({})".format(bytes2unicode(comment)) sourcestamp = dict( branch=branch, revision=revision, repository=repository, project=project, patch_level=patch[0], patch_body=patch[1], patch_subdir='', patch_author=who or '', patch_comment=comment or '', codebase='', ) # note: no way to specify patch subdir - #1769 requested_props = Properties() requested_props.update(properties, "try build") (bsid, brids) = yield self.scheduler.addBuildsetForSourceStamps( sourcestamps=[sourcestamp], reason=reason, properties=requested_props, builderNames=builderNames) # return a remotely-usable BuildSetStatus object bss = RemoteBuildSetStatus(self.scheduler.master, bsid, brids) return bss
def __init__(self, owner, slug, branch=None, pollInterval=10 * 60, useTimestamps=True, category=None, project='', pullrequest_filter=True, encoding='utf-8', pollAtLaunch=False ): self.owner = owner self.slug = slug self.branch = branch base.PollingChangeSource.__init__( self, name='/'.join([owner, slug]), pollInterval=pollInterval, pollAtLaunch=pollAtLaunch) self.encoding = encoding if hasattr(pullrequest_filter, '__call__'): self.pullrequest_filter = pullrequest_filter else: self.pullrequest_filter = (lambda _: pullrequest_filter) self.lastChange = time.time() self.lastPoll = time.time() self.useTimestamps = useTimestamps self.category = category if callable( category) else bytes2unicode(category) self.project = bytes2unicode(project) self.initLock = defer.DeferredLock()
def addHTMLLog(self, name, html): logid = yield self.master.data.updates.addLog(self.stepid, util.bytes2unicode(name), 'h') _log = self._newLog(name, 'h', logid) html = bytes2unicode(html) yield _log.addContent(html) yield _log.finish()
def decode_file(file): # git use octal char sequences in quotes when non ASCII match = re.match('^"(.*)"$', file) if match: file = bytes2unicode(match.groups()[0], encoding=self.encoding, errors='unicode_escape') return bytes2unicode(file, encoding=self.encoding)
def setProperty(self, name, value, source, runtime=False): name = util.bytes2unicode(name) json.dumps(value) # Let the exception propagate ... source = util.bytes2unicode(source) self.properties[name] = (value, source) if runtime: self.runtime.add(name)
def getResultSummary(self): src = bytes2unicode(self.src, errors='replace') dest = bytes2unicode(self.dest, errors='replace') copy = "{} to {}".format(src, dest) if self.results == SUCCESS: rv = 'Copied ' + copy else: rv = 'Copying ' + copy + ' failed.' return {'step': rv}
def __init__(self, repourl, branches=None, branch=None, workdir=None, pollInterval=10 * 60, gitbin='git', usetimestamps=True, category=None, project=None, pollinterval=-2, fetch_refspec=None, encoding='utf-8', name=None, pollAtLaunch=False, buildPushesWithNoCommits=False, only_tags=False, sshPrivateKey=None): # for backward compatibility; the parameter used to be spelled with 'i' if pollinterval != -2: pollInterval = pollinterval if name is None: name = repourl base.PollingChangeSource.__init__(self, name=name, pollInterval=pollInterval, pollAtLaunch=pollAtLaunch, sshPrivateKey=sshPrivateKey) if project is None: project = '' if only_tags and (branch or branches): config.error("GitPoller: can't specify only_tags and branch/branches") if branch and branches: config.error("GitPoller: can't specify both branch and branches") elif branch: branches = [branch] elif not branches: if only_tags: branches = lambda ref: ref.startswith('refs/tags/') # noqa: E731 else: branches = ['master'] self.repourl = repourl self.branches = branches self.encoding = encoding self.buildPushesWithNoCommits = buildPushesWithNoCommits self.gitbin = gitbin self.workdir = workdir self.usetimestamps = usetimestamps self.category = category if callable( category) else bytes2unicode(category, encoding=self.encoding) self.project = bytes2unicode(project, encoding=self.encoding) self.changeCount = 0 self.lastRev = {} self.sshPrivateKey = sshPrivateKey self.setupGit() if fetch_refspec is not None: config.error("GitPoller: fetch_refspec is no longer supported. " "Instead, only the given branches are downloaded.") if self.workdir is None: self.workdir = 'gitpoller-work'
def addHTMLLog(self, name, html): if self.stepid is None: raise BuildStepCancelled logid = yield self.master.data.updates.addLog(self.stepid, util.bytes2unicode(name), 'h') _log = self._newLog(name, 'h', logid) html = bytes2unicode(html) yield _log.addContent(html) yield _log.finish()
def setupWorkerBuildirProperty(self, workerforbuilder): path_module = workerforbuilder.worker.path_module # navigate our way back to the L{buildbot.worker.Worker} # object that came from the config, and get its properties if workerforbuilder.worker.worker_basedir: builddir = path_module.join( bytes2unicode(workerforbuilder.worker.worker_basedir), bytes2unicode(self.builder.config.workerbuilddir)) self.setProperty("builddir", builddir, "Worker")
def decode(x): if isinstance(x, bytes): return bytes2unicode(x) elif isinstance(x, (list, tuple)): return [bytes2unicode(y) for y in x] elif isinstance(x, dict): newArgs = {} for a, b in x.items(): newArgs[decode(a)] = decode(b) return newArgs return x
def __init__(self, repourl, split_file=None, svnuser=None, svnpasswd=None, pollInterval=10 * 60, histmax=100, svnbin='svn', revlinktmpl='', category=None, project='', cachepath=None, pollinterval=-2, extra_args=None, name=None, pollAtLaunch=False): # for backward compatibility; the parameter used to be spelled with 'i' if pollinterval != -2: pollInterval = pollinterval if name is None: name = repourl base.PollingChangeSource.__init__(self, name=name, pollInterval=pollInterval, pollAtLaunch=pollAtLaunch, svnuser=svnuser, svnpasswd=svnpasswd) if repourl.endswith("/"): repourl = repourl[:-1] # strip the trailing slash self.repourl = repourl self.extra_args = extra_args self.split_file = split_file or split_file_alwaystrunk self.svnuser = svnuser self.svnpasswd = svnpasswd self.revlinktmpl = revlinktmpl # include environment variables required for ssh-agent auth self.environ = os.environ.copy() self.svnbin = svnbin self.histmax = histmax self._prefix = None self.category = category if callable( category) else util.bytes2unicode(category) self.project = util.bytes2unicode(project) self.cachepath = cachepath if self.cachepath and os.path.exists(self.cachepath): try: with open(self.cachepath, "r") as f: self.last_change = int(f.read().strip()) log.msg("SVNPoller: SVNPoller(%s) setting last_change to %s" % ( self.repourl, self.last_change)) # try writing it, too with open(self.cachepath, "w") as f: f.write(str(self.last_change)) except Exception: self.cachepath = None log.msg(("SVNPoller: SVNPoller(%s) cache file corrupt or unwriteable; " + "skipping and not using") % self.repourl) log.err()
def test_getLogLines_bug3101(self): # regression test for #3101 content = self.bug3101Content yield self.insertTestData(self.backgroundData + self.bug3101Rows) # overall content is the same, with '\n' padding at the end expected = bytes2unicode(self.bug3101Content + b'\n') self.assertEqual((yield self.db.logs.getLogLines(1470, 0, 99)), expected) # try to fetch just one line expected = bytes2unicode(content.split(b'\n')[0] + b'\n') self.assertEqual((yield self.db.logs.getLogLines(1470, 0, 0)), expected)
def _processChanges(self, unused_output): """Send info about pulled changes to the master and record current. HgPoller does the recording by moving the working dir to the head of the branch. We don't update the tree (unnecessary treatment and waste of space) instead, we simply store the current rev number in a file. Recall that hg rev numbers are local and incremental. """ oid, current = yield self._getCurrentRev() # hg log on a range of revisions is never empty # also, if a numeric revision does not exist, a node may match. # Therefore, we have to check explicitly that branch head > current. head = yield self._getHead() if head is None: return elif current is not None and head <= current: return if current is None: # we could have used current = -1 convention as well (as hg does) revrange = '%d:%d' % (head, head) else: revrange = '%d:%s' % (current + 1, head) # two passes for hg log makes parsing simpler (comments is multi-lines) revListArgs = ['log', '-b', self.branch, '-r', revrange, r'--template={rev}:{node}\n'] results = yield utils.getProcessOutput(self.hgbin, revListArgs, path=self._absWorkdir(), env=os.environ, errortoo=False) results = results.decode(self.encoding) revNodeList = [rn.split(u':', 1) for rn in results.strip().split()] log.msg('hgpoller: processing %d changes: %r in %r' % (len(revNodeList), revNodeList, self._absWorkdir())) for rev, node in revNodeList: timestamp, author, files, comments = yield self._getRevDetails( node) yield self.master.data.updates.addChange( author=author, revision=text_type(node), files=files, comments=comments, when_timestamp=int(timestamp) if timestamp else None, branch=bytes2unicode(self.branch), category=bytes2unicode(self.category), project=bytes2unicode(self.project), repository=bytes2unicode(self.repourl), src=u'hg') # writing after addChange so that a rev is never missed, # but at once to avoid impact from later errors yield self._setCurrentRev(rev, oid=oid)
def _get_payload(self, request): content = request.content.read() content = bytes2unicode(content) content_type = request.getHeader(b'Content-Type') content_type = bytes2unicode(content_type) if content_type.startswith('application/json'): payload = json.loads(content) else: raise ValueError('Unknown content type: {}' .format(content_type)) log.msg("Payload: {}".format(payload)) return payload
def makeConfiguration(self, request): config = {} config.update(self.default) for k, v in self.ep.config.items(): if k == 'color_scheme': config[k].update(v) else: config[k] = v for k, v in request.args.items(): k = bytes2unicode(k) config[k] = escape(bytes2unicode(v[0])) return config
def addChange(self, who=None, files=None, comments=None, **kwargs): # deprecated in 0.9.0; will be removed in 1.0.0 log.msg("WARNING: change source is using deprecated " "self.master.addChange method; this method will disappear in " "Buildbot-1.0.0") # handle positional arguments kwargs['who'] = who kwargs['files'] = files kwargs['comments'] = comments def handle_deprec(oldname, newname): if oldname in kwargs: old = kwargs.pop(oldname) if old is not None: if kwargs.get(newname) is None: log.msg("WARNING: change source is using deprecated " "addChange parameter '%s'" % oldname) return old raise TypeError("Cannot provide '%s' and '%s' to addChange" % (oldname, newname)) return kwargs.get(newname) kwargs['author'] = handle_deprec("who", "author") kwargs['when_timestamp'] = handle_deprec("when", "when_timestamp") # timestamp must be an epoch timestamp now if isinstance(kwargs.get('when_timestamp'), datetime.datetime): kwargs['when_timestamp'] = datetime2epoch(kwargs['when_timestamp']) # unicodify stuff for k in ('comments', 'author', 'revision', 'branch', 'category', 'revlink', 'repository', 'codebase', 'project'): if k in kwargs: kwargs[k] = bytes2unicode(kwargs[k]) if kwargs.get('files'): kwargs['files'] = [bytes2unicode(f) for f in kwargs['files']] if kwargs.get('properties'): kwargs['properties'] = dict((bytes2unicode(k), v) for k, v in iteritems(kwargs['properties'])) # pass the converted call on to the data API changeid = yield self.data.updates.addChange(**kwargs) # and turn that changeid into a change object, since that's what # callers expected (and why this method was deprecated) chdict = yield self.db.changes.getChange(changeid) change = yield changes.Change.fromChdict(self, chdict) defer.returnValue(change)
def _convert_nonzero_to_failure(res, command, args, path): "utility to handle the result of getProcessOutputAndValue" (stdout, stderr, code) = res stdout = bytes2unicode(stdout, self.encoding) stderr = bytes2unicode(stderr, self.encoding) if code != 0: if code == 128: raise GitError('command %s %s in %s on repourl %s failed with exit code %d: %s' % (command, args, path, self.repourl, code, stderr)) raise EnvironmentError('command %s %s in %s on repourl %s failed with exit code %d: %s' % (command, args, path, self.repourl, code, stderr)) return stdout.strip()
def _get_payload(self, request): content = request.content.read() content = bytes2unicode(content) signature = request.getHeader(_HEADER_SIGNATURE) signature = bytes2unicode(signature) if not signature and self._strict: raise ValueError('Request has no required signature') if self._secret and signature: try: hash_type, hexdigest = signature.split('=') except ValueError: raise ValueError( 'Wrong signature format: {}'.format(signature)) if hash_type != 'sha1': raise ValueError('Unknown hash type: {}'.format(hash_type)) mac = hmac.new(unicode2bytes(self._secret), msg=unicode2bytes(content), digestmod=sha1) def _cmp(a, b): try: # try the more secure compare_digest() first from hmac import compare_digest return compare_digest(a, b) except ImportError: # pragma: no cover # and fallback to the insecure simple comparison otherwise return a == b if not _cmp(bytes2unicode(mac.hexdigest()), hexdigest): raise ValueError('Hash mismatch') content_type = request.getHeader(b'Content-Type') if content_type == b'application/json': payload = json.loads(content) elif content_type == b'application/x-www-form-urlencoded': payload = json.loads(bytes2unicode(request.args[b'payload'][0])) else: raise ValueError('Unknown content type: {}'.format(content_type)) log.msg("Payload: {}".format(payload), logLevel=logging.DEBUG) return payload
def handleJobFile(self, filename, f): try: parsed_job = self.parseJob(f) builderNames = parsed_job['builderNames'] except BadJobfile: log.msg("%s reports a bad jobfile in %s" % (self, filename)) log.err() return defer.succeed(None) # Validate/fixup the builder names. builderNames = self.filterBuilderList(builderNames) if not builderNames: log.msg( "incoming Try job did not specify any allowed builder names") return defer.succeed(None) who = "" if parsed_job['who']: who = parsed_job['who'] comment = "" if parsed_job['comment']: comment = parsed_job['comment'] sourcestamp = dict(branch=parsed_job['branch'], codebase='', revision=parsed_job['baserev'], patch_body=parsed_job['patch_body'], patch_level=parsed_job['patch_level'], patch_author=who, patch_comment=comment, # TODO: can't set this remotely - #1769 patch_subdir='', project=parsed_job['project'], repository=parsed_job['repository']) reason = "'try' job" if parsed_job['who']: reason += " by user {}".format(bytes2unicode(parsed_job['who'])) properties = parsed_job['properties'] requested_props = Properties() requested_props.update(properties, "try build") return self.addBuildsetForSourceStamps( sourcestamps=[sourcestamp], reason=reason, external_idstring=bytes2unicode(parsed_job['jobid']), builderNames=builderNames, properties=requested_props)
def getChanges(self, request): """ Take the logic from the change hook, and then delegate it to the proper handler We use the buildbot plugin mechanisms to find out about dialects and call getChanges() the return value is a list of changes if DIALECT is unspecified, a sample implementation is provided """ uriRE = re.search(r'^/change_hook/?([a-zA-Z0-9_]*)', bytes2unicode(request.uri)) if not uriRE: log.msg("URI doesn't match change_hook regex: %s" % request.uri) raise ValueError( "URI doesn't match change_hook regex: %s" % request.uri) changes = [] src = None # Was there a dialect provided? if uriRE.group(1): dialect = uriRE.group(1) else: dialect = 'base' handler = self.makeHandler(dialect) changes, src = yield handler.getChanges(request) return (changes, src)
def getEndpoint(self, request, method, params): # note that trailing slashes are not allowed request_postpath = tuple(bytes2unicode(p) for p in request.postpath) yield self.master.www.assertUserAllowed(request, request_postpath, method, params) ret = yield self.master.data.getEndpoint(request_postpath) return ret
def checkFields(fields, negOk=False): for field in fields: k = bytes2unicode(field) if k[0] == '-' and negOk: k = k[1:] if k not in entityType.fieldNames: raise BadRequest("no such field '{}'".format(k))
def testBasicWorker2_16(self): s = transfer.JSONPropertiesDownload("props.json") s.build = Mock() props = Properties() props.setProperty('key1', 'value1', 'test') s.build.getProperties.return_value = props s.build.getWorkerCommandVersion.return_value = '2.16' ss = Mock() ss.asDict.return_value = dict(revision="12345") s.build.getAllSourceStamps.return_value = [ss] s.worker = Mock() s.remote = Mock() s.start() for c in s.remote.method_calls: name, command, args = c commandName = command[3] kwargs = command[-1] if commandName == 'downloadFile': self.assertEqual(kwargs['slavedest'], 'props.json') reader = kwargs['reader'] data = reader.remote_read(100) data = bytes2unicode(data) actualJson = json.loads(data) expectedJson = dict(sourcestamps=[ss.asDict()], properties={'key1': 'value1'}) self.assertEqual(actualJson, expectedJson) break else: raise ValueError("No downloadFile command found")
def requestAvatar(self, username, mind, interface): assert interface == pb.IPerspective username = bytes2unicode(username) if username not in self.users: d = defer.succeed(None) # no perspective else: _, afactory = self.users.get(username) d = defer.maybeDeferred(afactory, mind, username) # check that we got a perspective @d.addCallback def check(persp): if not persp: raise ValueError("no perspective for '%s'" % username) return persp # call the perspective's attached(mind) @d.addCallback def call_attached(persp): d = defer.maybeDeferred(persp.attached, mind) d.addCallback(lambda _: persp) # keep returning the perspective return d # return the tuple requestAvatar is expected to return @d.addCallback def done(persp): return (pb.IPerspective, persp, lambda: persp.detached(mind)) return d
def stepDone(self, results, step): """This method is called when the BuildStep completes. It is passed a status object from the BuildStep and is responsible for merging the Step's results into those of the overall Build.""" terminate = False text = None if isinstance(results, tuple): results, text = results assert isinstance(results, type(SUCCESS)), "got %r" % (results,) summary = yield step.getBuildResultSummary() if 'build' in summary: text = [summary['build']] log.msg(" step '%s' complete: %s (%s)" % (step.name, statusToString(results), text)) if text: self.text.extend(text) self.master.data.updates.setBuildStateString(self.buildid, bytes2unicode(" ".join(self.text))) self.results, terminate = computeResultAndTermination(step, results, self.results) if not self.conn: # force the results to retry if the connection was lost self.results = RETRY terminate = True defer.returnValue(terminate)
def test_createEmail_message_with_patch_and_log_containing_unicode(self): _, builds = yield self.setupBuildResults(SUCCESS) msgdict = create_msgdict() patches = [{'body': u'\u00E5\u00E4\u00F6'}] logs = yield self.master.data.get(("steps", 50, 'logs')) for l in logs: l['stepname'] = "fakestep" l['content'] = yield self.master.data.get(("logs", l['logid'], 'contents')) mn = yield self.setupMailNotifier('*****@*****.**', addLogs=True) m = yield mn.createEmail(msgdict, u'builder-n\u00E5me', u'project-n\u00E5me', SUCCESS, builds, patches, logs) try: s = m.as_string() # python 2.6 default transfer in base64 for utf-8 if "base64" not in s: self.assertIn("Unicode log", s) else: # b64encode and remove '=' padding (hence [:-1]) logStr = bytes2unicode(base64.b64encode(b"Unicode log")[:-1]) self.assertIn(logStr, s) self.assertIn( 'Content-Disposition: attachment; filename="fakestep.stdio"', s) except UnicodeEncodeError: self.fail('Failed to call as_string() on email message.')
def __init__(self, repourl, branch='default', workdir=None, pollInterval=10 * 60, hgbin='hg', usetimestamps=True, category=None, project='', pollinterval=-2, encoding='utf-8', name=None, pollAtLaunch=False): # for backward compatibility; the parameter used to be spelled with 'i' if pollinterval != -2: pollInterval = pollinterval if name is None: name = "%s[%s]" % (repourl, branch) self.repourl = repourl self.branch = branch base.PollingChangeSource.__init__( self, name=name, pollInterval=pollInterval, pollAtLaunch=pollAtLaunch) self.encoding = encoding self.lastChange = time.time() self.lastPoll = time.time() self.hgbin = hgbin self.workdir = workdir self.usetimestamps = usetimestamps self.category = category if callable( category) else bytes2unicode(category) self.project = project self.commitInfo = {} self.initLock = defer.DeferredLock() if self.workdir is None: config.error("workdir is mandatory for now in HgPoller")
def render_GET(self, request): def decode(x): if isinstance(x, bytes): return bytes2unicode(x) elif isinstance(x, (list, tuple)): return [bytes2unicode(y) for y in x] elif isinstance(x, dict): newArgs = {} for a, b in x.items(): newArgs[decode(a)] = decode(b) return newArgs return x args = decode(request.args) content_type = request.getHeader(b'content-type') if content_type == b"application/json": jsonBytes = request.content.read() jsonStr = bytes2unicode(jsonBytes) args['json_received'] = json.loads(jsonStr) data = json.dumps(args) data = unicode2bytes(data) request.setHeader(b'content-type', b'application/json') request.setHeader(b'content-length', intToBytes(len(data))) if request.method == b'HEAD': return b'' return data
def thd(): url = self.tokenUri data = {'redirect_uri': self.loginUri, 'code': code, 'grant_type': self.grantType} auth = None if self.getTokenUseAuthHeaders: auth = (self.clientId, self.clientSecret) else: data.update( {'client_id': self.clientId, 'client_secret': self.clientSecret}) data.update(self.tokenUriAdditionalParams) response = requests.post( url, data=data, auth=auth, verify=self.sslVerify) response.raise_for_status() responseContent = bytes2unicode(response.content) try: content = json.loads(responseContent) except ValueError: content = parse_qs(responseContent) for k, v in iteritems(content): content[k] = v[0] except TypeError: content = responseContent session = self.createSessionFromToken(content) return self.getUserInfoFromOAuthClient(session)
def test_nonascii(self): self.assertRaises( UnicodeDecodeError, lambda: util.bytes2unicode(b'a\x85', encoding='ascii'))
def test_bytes2unicode(self): rv1 = util.bytes2unicode(b'abcd') rv2 = util.bytes2unicode('efgh') self.assertEqual(type(rv1), text_type) self.assertEqual(type(rv2), text_type)
def create_changes(self, new_logentries): changes = [] for el in new_logentries: revision = str(el.getAttribute("revision")) revlink = '' if self.revlinktmpl and revision: revlink = self.revlinktmpl % urlquote_plus(revision) revlink = str(revlink) log.msg(f"Adding change revision {revision}") author = self._get_text(el, "author") comments = self._get_text(el, "msg") # there is a "date" field, but it provides localtime in the # repository's timezone, whereas we care about buildmaster's # localtime (since this will get used to position the boxes on # the Waterfall display, etc). So ignore the date field, and # addChange will fill in with the current time branches = {} try: pathlist = el.getElementsByTagName("paths")[0] except IndexError: # weird, we got an empty revision log.msg("ignoring commit with no paths") continue for p in pathlist.getElementsByTagName("path"): kind = p.getAttribute("kind") action = p.getAttribute("action") path = "".join([t.data for t in p.childNodes]) if path.startswith("/"): path = path[1:] if kind == "dir" and not path.endswith("/"): path += "/" where = self._transform_path(path) # if 'where' is None, the file was outside any project that # we care about and we should ignore it if where: branch = where.get("branch", None) filename = where["path"] if branch not in branches: branches[branch] = { 'files': [], 'number_of_directories': 0} if filename == "": # root directory of branch branches[branch]['files'].append(filename) branches[branch]['number_of_directories'] += 1 elif filename.endswith("/"): # subdirectory of branch branches[branch]['files'].append(filename[:-1]) branches[branch]['number_of_directories'] += 1 else: branches[branch]['files'].append(filename) if "action" not in branches[branch]: branches[branch]['action'] = action for key in ("repository", "project", "codebase"): if key in where: branches[branch][key] = where[key] for branch, info in branches.items(): action = info['action'] files = info['files'] number_of_directories_changed = info['number_of_directories'] number_of_files_changed = len(files) if (action == 'D' and number_of_directories_changed == 1 and number_of_files_changed == 1 and files[0] == ''): log.msg(f"Ignoring deletion of branch '{branch}'") else: chdict = dict( author=author, committer=None, # weakly assume filenames are utf-8 files=[bytes2unicode(f, 'utf-8', 'replace') for f in files], comments=comments, revision=revision, branch=util.bytes2unicode(branch), revlink=revlink, category=self.category, repository=util.bytes2unicode( info.get('repository', self.repourl)), project=util.bytes2unicode( info.get('project', self.project)), codebase=util.bytes2unicode( info.get('codebase', None))) changes.append(chdict) return changes
def thd(): res = requests.get('http://localhost:5000/auth/login') content = bytes2unicode(res.content) webbrowser.open(content)
def parseJob(self, f): # jobfiles are serialized build requests. Each is a list of # serialized netstrings, in the following order: # format version number: # "1" the original # "2" introduces project and repository # "3" introduces who # "4" introduces comment # "5" introduces properties and JSON serialization of values after # version # jobid: arbitrary string, used to find the buildSet later # branch: branch name, "" for default-branch # baserev: revision, "" for HEAD # patch_level: usually "1" # patch_body: patch to be applied for build # repository # project # who: user requesting build # comment: comment from user about diff and/or build # builderNames: list of builder names # properties: dict of build properties p = netstrings.NetstringParser() f.seek(0, 2) if f.tell() > basic.NetstringReceiver.MAX_LENGTH: raise BadJobfile( "The patch size is greater that NetStringReceiver.MAX_LENGTH. " "Please Set this higher in the master.cfg") f.seek(0, 0) try: p.feed(f.read()) except basic.NetstringParseError as e: raise BadJobfile("unable to parse netstrings") from e if not p.strings: raise BadJobfile("could not find any complete netstrings") ver = bytes2unicode(p.strings.pop(0)) v1_keys = ['jobid', 'branch', 'baserev', 'patch_level', 'patch_body'] v2_keys = v1_keys + ['repository', 'project'] v3_keys = v2_keys + ['who'] v4_keys = v3_keys + ['comment'] keys = [v1_keys, v2_keys, v3_keys, v4_keys] # v5 introduces properties and uses JSON serialization parsed_job = {} def extract_netstrings(p, keys): for i, key in enumerate(keys): parsed_job[key] = bytes2unicode(p.strings[i]) def postprocess_parsed_job(): # apply defaults and handle type casting parsed_job['branch'] = parsed_job['branch'] or None parsed_job['baserev'] = parsed_job['baserev'] or None parsed_job['patch_level'] = int(parsed_job['patch_level']) for key in 'repository project who comment'.split(): parsed_job[key] = parsed_job.get(key, '') parsed_job['properties'] = parsed_job.get('properties', {}) if ver <= "4": i = int(ver) - 1 extract_netstrings(p, keys[i]) parsed_job['builderNames'] = [ bytes2unicode(s) for s in p.strings[len(keys[i]):] ] postprocess_parsed_job() elif ver == "5": try: data = bytes2unicode(p.strings[0]) parsed_job = json.loads(data) except ValueError as e: raise BadJobfile("unable to parse JSON") from e postprocess_parsed_job() else: raise BadJobfile("unknown version '{}'".format(ver)) return parsed_job
def valueFromString(self, arg): val = util.bytes2unicode(arg) if not self.identRe.match(val) or not 0 < len(val) <= self.len: raise TypeError return val
def addHeader(self, data): data = bytes2unicode(data) self.chunks.append((self.HEADER, data)) self._delay(lambda: self.asyncLogfile.addHeader(data))
def decodeResultSpec(self, request, endpoint): reqArgs = request.args def checkFields(fields, negOk=False): for field in fields: k = bytes2unicode(field) if k[0] == '-' and negOk: k = k[1:] if k not in entityType.fieldNames: raise BadRequest("no such field '{}'".format(k)) entityType = endpoint.rtype.entityType limit = offset = order = fields = None filters, properties = [], [] for arg in reqArgs: argStr = bytes2unicode(arg) if arg == b'order': order = tuple([bytes2unicode(o) for o in reqArgs[arg]]) checkFields(order, True) elif arg == b'field': fields = reqArgs[arg] checkFields(fields, False) elif arg == b'limit': try: limit = int(reqArgs[arg][0]) except Exception: raise BadRequest('invalid limit') elif arg == b'offset': try: offset = int(reqArgs[arg][0]) except Exception: raise BadRequest('invalid offset') elif arg == b'property': try: props = [] for v in reqArgs[arg]: if not isinstance(v, (bytes, str)): raise TypeError("Invalid type {} for {}".format( type(v), v)) props.append(bytes2unicode(v)) except Exception: raise BadRequest( 'invalid property value for {}'.format(arg)) properties.append(resultspec.Property(arg, 'eq', props)) elif argStr in entityType.fieldNames: field = entityType.fields[argStr] try: values = [field.valueFromString(v) for v in reqArgs[arg]] except Exception: raise BadRequest( 'invalid filter value for {}'.format(argStr)) filters.append(resultspec.Filter(argStr, 'eq', values)) elif '__' in argStr: field, op = argStr.rsplit('__', 1) args = reqArgs[arg] operators = (resultspec.Filter.singular_operators if len(args) == 1 else resultspec.Filter.plural_operators) if op in operators and field in entityType.fieldNames: fieldType = entityType.fields[field] try: values = [ fieldType.valueFromString(v) for v in reqArgs[arg] ] except Exception: raise BadRequest( 'invalid filter value for {}'.format(argStr)) filters.append(resultspec.Filter(field, op, values)) else: raise BadRequest( "unrecognized query parameter '{}'".format(argStr)) # if ordering or filtering is on a field that's not in fields, bail out if fields: fields = [bytes2unicode(f) for f in fields] fieldsSet = set(fields) if order and {o.lstrip('-') for o in order} - fieldsSet: raise BadRequest("cannot order on un-selected fields") for filter in filters: if filter.field not in fieldsSet: raise BadRequest("cannot filter on un-selected fields") # build the result spec rspec = resultspec.ResultSpec(fields=fields, limit=limit, offset=offset, order=order, filters=filters, properties=properties) # for singular endpoints, only allow fields if not endpoint.isCollection: if rspec.filters: raise BadRequest("this is not a collection") return rspec
def __init__(self, name=None, workername=None, workernames=None, builddir=None, workerbuilddir=None, factory=None, tags=None, nextWorker=None, nextBuild=None, locks=None, env=None, properties=None, collapseRequests=None, description=None, canStartBuild=None, defaultProperties=None ): # name is required, and can't start with '_' if not name or type(name) not in (bytes, str): error("builder's name is required") name = '<unknown>' elif name[0] == '_' and name not in RESERVED_UNDERSCORE_NAMES: error("builder names must not start with an underscore: '{}'".format(name)) try: self.name = util.bytes2unicode(name, encoding="ascii") except UnicodeDecodeError: error("builder names must be unicode or ASCII") # factory is required if factory is None: error("builder '{}' has no factory".format(name)) from buildbot.process.factory import BuildFactory if factory is not None and not isinstance(factory, BuildFactory): error("builder '{}'s factory is not a BuildFactory instance".format(name)) self.factory = factory # workernames can be a single worker name or a list, and should also # include workername, if given if isinstance(workernames, str): workernames = [workernames] if workernames: if not isinstance(workernames, list): error("builder '{}': workernames must be a list or a string".format(name)) else: workernames = [] if workername: if not isinstance(workername, str): error(("builder '{}': workername must be a string but it is {}" ).format(name, repr(workername))) workernames = workernames + [workername] if not workernames: error("builder '{}': at least one workername is required".format(name)) self.workernames = workernames # builddir defaults to name if builddir is None: builddir = safeTranslate(name) builddir = bytes2unicode(builddir) self.builddir = builddir # workerbuilddir defaults to builddir if workerbuilddir is None: workerbuilddir = builddir self.workerbuilddir = workerbuilddir # remainder are optional if tags: if not isinstance(tags, list): error("builder '{}': tags must be a list".format(name)) bad_tags = any((tag for tag in tags if not isinstance(tag, str))) if bad_tags: error( "builder '{}': tags list contains something that is not a string".format(name)) if len(tags) != len(set(tags)): dupes = " ".join({x for x in tags if tags.count(x) > 1}) error( "builder '{}': tags list contains duplicate tags: {}".format(name, dupes)) else: tags = [] self.tags = tags self.nextWorker = nextWorker if nextWorker and not callable(nextWorker): error('nextWorker must be a callable') self.nextBuild = nextBuild if nextBuild and not callable(nextBuild): error('nextBuild must be a callable') self.canStartBuild = canStartBuild if canStartBuild and not callable(canStartBuild): error('canStartBuild must be a callable') self.locks = locks or [] self.env = env or {} if not isinstance(self.env, dict): error("builder's env must be a dictionary") self.properties = properties or {} self.defaultProperties = defaultProperties or {} self.collapseRequests = collapseRequests self.description = description
def sendComment(self, pr_url, text): path = urlparse(unicode2bytes(pr_url)).path payload = {'text': text} return self._http.post( COMMENT_API_URL.format(path=bytes2unicode(path)), json=payload)
def setHost(self, host): self.host = bytes2unicode(host)
def __init__(self, p4port=None, p4user=None, p4passwd=None, p4base='//', p4bin='p4', split_file=lambda branchfile: (None, branchfile), pollInterval=60 * 10, histmax=None, pollinterval=-2, encoding='utf8', project=None, name=None, use_tickets=False, ticket_login_interval=60 * 60 * 24, server_tz=None, pollAtLaunch=False, revlink=lambda branch, revision: (u''), resolvewho=lambda who: (who)): # for backward compatibility; the parameter used to be spelled with 'i' if pollinterval != -2: pollInterval = pollinterval if name is None: name = "P4Source:%s:%s" % (p4port, p4base) base.PollingChangeSource.__init__(self, name=name, pollInterval=pollInterval, pollAtLaunch=pollAtLaunch) if project is None: project = '' if use_tickets and not p4passwd: config.error( "You need to provide a P4 password to use ticket authentication" ) if not callable(revlink): config.error("You need to provide a valid callable for revlink") if not callable(resolvewho): config.error("You need to provide a valid callable for resolvewho") self.p4port = p4port self.p4user = p4user self.p4passwd = p4passwd self.p4base = p4base self.p4bin = p4bin self.split_file = split_file self.encoding = encoding self.project = util.bytes2unicode(project) self.use_tickets = use_tickets self.ticket_login_interval = ticket_login_interval self.revlink_callable = revlink self.resolvewho_callable = resolvewho self.server_tz = dateutil.tz.gettz(server_tz) if server_tz else None if server_tz is not None and self.server_tz is None: raise P4PollerError( "Failed to get timezone from server_tz string '{}'".format( server_tz)) self._ticket_passwd = None self._ticket_login_counter = 0
def test_unicode(self): rv = util.bytes2unicode(u'\N{SNOWMAN}', encoding='ascii') self.assertEqual((rv, type(rv)), (u'\N{SNOWMAN}', text_type))
def setAdmin(self, admin): self.admin = bytes2unicode(admin)
def addStderr(self, data): data = bytes2unicode(data) self.chunks.append((self.STDERR, data)) self._delay(lambda: self.asyncLogfile.addStderr(data))
def renderRest(self, request): def writeError(msg, errcode=404, jsonrpccode=None): if self.debug: log.msg("REST error: {}".format(msg)) request.setResponseCode(errcode) request.setHeader(b'content-type', b'text/plain; charset=utf-8') msg = bytes2unicode(msg) data = json.dumps(dict(error=msg)) data = unicode2bytes(data) request.write(data) with self.handleErrors(writeError): ep, kwargs = yield self.getEndpoint(request, bytes2unicode(request.method), {}) rspec = self.decodeResultSpec(request, ep) data = yield ep.get(rspec, kwargs) if data is None: msg = ("not found while getting from {} with " "arguments {} and {}").format(repr(ep), repr(rspec), str(kwargs)) msg = unicode2bytes(msg) writeError(msg, errcode=404) return if ep.isRaw: self.encodeRaw(data, request) return # post-process any remaining parts of the resultspec data = rspec.apply(data) # annotate the result with some metadata meta = {} if ep.isCollection: offset, total = data.offset, data.total if offset is None: offset = 0 # add total, if known if total is not None: meta['total'] = total # get the real list instance out of the ListResult data = data.data else: data = [data] typeName = ep.rtype.plural data = {typeName: data, 'meta': meta} # set up the content type and formatting options; if the request # accepts text/html or text/plain, the JSON will be rendered in a # readable, multiline format. if b'application/json' in (request.getHeader(b'accept') or b''): compact = True request.setHeader(b"content-type", b'application/json; charset=utf-8') else: compact = False request.setHeader(b"content-type", b'text/plain; charset=utf-8') # set up caching if self.cache_seconds: now = datetime.datetime.utcnow() expires = now + datetime.timedelta(seconds=self.cache_seconds) expiresBytes = unicode2bytes( expires.strftime("%a, %d %b %Y %H:%M:%S GMT")) request.setHeader(b"Expires", expiresBytes) request.setHeader(b"Pragma", b"no-cache") # filter out blanks if necessary and render the data if compact: data = json.dumps(data, default=toJson, sort_keys=True, separators=(',', ':')) else: data = json.dumps(data, default=toJson, sort_keys=True, indent=2) if request.method == b"HEAD": request.setHeader(b"content-length", unicode2bytes(str(len(data)))) else: data = unicode2bytes(data) request.write(data)
def valueFromString(self, arg): val = util.bytes2unicode(arg) return val
def gettype(self): mimetype, options = cgi.parse_header(bytes2unicode(self.typeheader)) return mimetype
def parse(self, m, prefix=None): """Parse messages sent by the 'buildbot-cvs-mail' program. """ # The mail is sent from the person doing the checkin. Assume that the # local username is enough to identify them (this assumes a one-server # cvs-over-rsh environment rather than the server-dirs-shared-over-NFS # model) _, addr = parseaddr(m["from"]) if not addr: # no From means this message isn't from buildbot-cvs-mail return None at = addr.find("@") if at == -1: author = addr # might still be useful else: author = addr[:at] author = util.bytes2unicode(author, encoding="ascii") # CVS accepts RFC822 dates. buildbot-cvs-mail adds the date as # part of the mail header, so use that. # This assumes cvs is being access via ssh or pserver, so the time # will be the CVS server's time. # calculate a "revision" based on that timestamp, or the current time # if we're unable to parse the date. log.msg('Processing CVS mail') dateTuple = parsedate_tz(m["date"]) if dateTuple is None: when = util.now() else: when = mktime_tz(dateTuple) theTime = datetime.datetime.utcfromtimestamp(float(when)) rev = theTime.strftime('%Y-%m-%d %H:%M:%S') catRE = re.compile(r'^Category:\s*(\S.*)') cvsRE = re.compile(r'^CVSROOT:\s*(\S.*)') cvsmodeRE = re.compile(r'^Cvsmode:\s*(\S.*)') filesRE = re.compile(r'^Files:\s*(\S.*)') modRE = re.compile(r'^Module:\s*(\S.*)') pathRE = re.compile(r'^Path:\s*(\S.*)') projRE = re.compile(r'^Project:\s*(\S.*)') singleFileRE = re.compile(r'(.*) (NONE|\d(\.|\d)+) (NONE|\d(\.|\d)+)') tagRE = re.compile(r'^\s+Tag:\s*(\S.*)') updateRE = re.compile(r'^Update of:\s*(\S.*)') comments = "" branch = None cvsroot = None fileList = None files = [] isdir = 0 path = None project = None lines = list(body_line_iterator(m)) while lines: line = lines.pop(0) m = catRE.match(line) if m: category = m.group(1) continue m = cvsRE.match(line) if m: cvsroot = m.group(1) continue m = cvsmodeRE.match(line) if m: cvsmode = m.group(1) continue m = filesRE.match(line) if m: fileList = m.group(1) continue m = modRE.match(line) if m: # We don't actually use this # module = m.group(1) continue m = pathRE.match(line) if m: path = m.group(1) continue m = projRE.match(line) if m: project = m.group(1) continue m = tagRE.match(line) if m: branch = m.group(1) continue m = updateRE.match(line) if m: # We don't actually use this # updateof = m.group(1) continue if line == "Log Message:\n": break # CVS 1.11 lists files as: # repo/path file,old-version,new-version file2,old-version,new-version # Version 1.12 lists files as: # file1 old-version new-version file2 old-version new-version # # files consists of tuples of 'file-name old-version new-version' # The versions are either dotted-decimal version numbers, ie 1.1 # or NONE. New files are of the form 'NONE NUMBER', while removed # files are 'NUMBER NONE'. 'NONE' is a literal string # Parsing this instead of files list in 'Added File:' etc # makes it possible to handle files with embedded spaces, though # it could fail if the filename was 'bad 1.1 1.2' # For cvs version 1.11, we expect # my_module new_file.c,NONE,1.1 # my_module removed.txt,1.2,NONE # my_module modified_file.c,1.1,1.2 # While cvs version 1.12 gives us # new_file.c NONE 1.1 # removed.txt 1.2 NONE # modified_file.c 1.1,1.2 if fileList is None: log.msg('CVSMaildirSource Mail with no files. Ignoring') return None # We don't have any files. Email not from CVS if cvsmode == '1.11': # Please, no repo paths with spaces! m = re.search('([^ ]*) ', fileList) if m: path = m.group(1) else: log.msg( 'CVSMaildirSource can\'t get path from file list. Ignoring mail' ) return None fileList = fileList[len(path):].strip() singleFileRE = re.compile( r'(.+?),(NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+)),(NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+))(?: |$)') # noqa pylint: disable=line-too-long elif cvsmode == '1.12': singleFileRE = re.compile( r'(.+?) (NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+)) (NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+))(?: |$)') # noqa pylint: disable=line-too-long if path is None: raise ValueError( 'CVSMaildirSource cvs 1.12 require path. Check cvs loginfo config' ) else: raise ValueError(f'Expected cvsmode 1.11 or 1.12. got: {cvsmode}') log.msg(f"CVSMaildirSource processing filelist: {fileList}") while fileList: m = singleFileRE.match(fileList) if m: curFile = path + '/' + m.group(1) files.append(curFile) fileList = fileList[m.end():] else: log.msg('CVSMaildirSource no files matched regex. Ignoring') return None # bail - we couldn't parse the files that changed # Now get comments while lines: line = lines.pop(0) comments += line comments = comments.rstrip() + "\n" if comments == '\n': comments = None return ('cvs', dict(author=author, committer=None, files=files, comments=comments, isdir=isdir, when=when, branch=branch, revision=rev, category=category, repository=cvsroot, project=project, properties=self.properties))
def extract_netstrings(p, keys): for i, key in enumerate(keys): if key == 'patch_body': parsed_job[key] = p.strings[i] else: parsed_job[key] = bytes2unicode(p.strings[i])
def extract_netstrings(p, keys): for i, key in enumerate(keys): parsed_job[key] = bytes2unicode(p.strings[i])
def getBuilderIdForName(self, name): # buildbot.config should ensure this is already unicode, but it doesn't # hurt to check again name = bytes2unicode(name) return self.master.data.updates.findBuilderId(name)
def match(self, ep, action="", options=None): if bytes2unicode(action).lower() != "get": return defer.succeed(Match(self.master)) return defer.succeed(None)
def _processChanges(self, page): result = json.loads(page, encoding=self.encoding) for pr in result['values']: branch = pr['source']['branch']['name'] nr = int(pr['id']) # Note that this is a short hash. The full length hash can be accessed via the # commit api resource but we want to avoid requesting multiple pages as long as # we are not sure that the pull request is new or updated. revision = pr['source']['commit']['hash'] # check branch if not self.branch or branch in self.branch: current = yield self._getCurrentRev(nr) # compare _short_ hashes to check if the PR has been updated if not current or current[0:12] != revision[0:12]: # parse pull request api page (required for the filter) page = yield client.getPage(str(pr['links']['self']['href'])) pr_json = json.loads(page, encoding=self.encoding) # filter pull requests by user function if not self.pullrequest_filter(pr_json): log.msg('pull request does not match filter') continue # access additional information author = pr['author']['display_name'] prlink = pr['links']['html']['href'] # Get time updated time. Note that the timezone offset is # ignored. if self.useTimestamps: updated = datetime.strptime( pr['updated_on'].split('.')[0], '%Y-%m-%dT%H:%M:%S') else: updated = epoch2datetime(reactor.seconds()) title = pr['title'] # parse commit api page page = yield client.getPage(str(pr['source']['commit']['links']['self']['href'])) commit_json = json.loads(page, encoding=self.encoding) # use the full-length hash from now on revision = commit_json['hash'] revlink = commit_json['links']['html']['href'] # parse repo api page page = yield client.getPage(str(pr['source']['repository']['links']['self']['href'])) repo_json = json.loads(page, encoding=self.encoding) repo = repo_json['links']['html']['href'] # update database yield self._setCurrentRev(nr, revision) # emit the change yield self.master.data.updates.addChange( author=bytes2unicode(author), revision=bytes2unicode(revision), revlink=bytes2unicode(revlink), comments=u'pull-request #%d: %s\n%s' % ( nr, title, prlink), when_timestamp=datetime2epoch(updated), branch=bytes2unicode(branch), category=self.category, project=self.project, repository=bytes2unicode(repo), src=u'bitbucket', )
def _poll(self): if self.use_tickets: self._ticket_login_counter -= 1 if self._ticket_login_counter <= 0: # Re-acquire the ticket and reset the counter. log.msg("P4Poller: (re)acquiring P4 ticket for %s..." % self.p4base) protocol = TicketLoginProtocol( self.p4passwd + "\n", self.p4base) self._acquireTicket(protocol) yield protocol.deferred self._ticket_passwd = self._parseTicketPassword( protocol.stdout) self._ticket_login_counter = max( self.ticket_login_interval / self.pollInterval, 1) if debug_logging: log.msg("P4Poller: got ticket password: %s" % self._ticket_passwd) log.msg( "P4Poller: next ticket acquisition in %d polls" % self._ticket_login_counter) args = [] if self.p4port: args.extend(['-p', self.p4port]) if self.p4user: args.extend(['-u', self.p4user]) if self.p4passwd: args.extend(['-P', self._getPasswd()]) args.extend(['changes']) if self.last_change is not None: args.extend( ['%s...@%d,#head' % (self.p4base, self.last_change + 1)]) else: args.extend(['-m', '1', '%s...' % (self.p4base,)]) result = yield self._get_process_output(args) # decode the result from its designated encoding try: result = bytes2unicode(result, self.encoding) except UnicodeError as ex: log.msg(u"{}: cannot fully decode {} in {}".format( ex, repr(result), self.encoding)) result = bytes2unicode(result, encoding=self.encoding, errors="replace") last_change = self.last_change changelists = [] for line in result.split('\n'): line = line.strip() if not line: continue m = self.changes_line_re.match(line) if not m: raise P4PollerError( "Unexpected 'p4 changes' output: %r" % result) num = int(m.group('num')) if last_change is None: # first time through, the poller just gets a "baseline" for where to # start on the next poll log.msg('P4Poller: starting at change %d' % num) self.last_change = num return changelists.append(num) changelists.reverse() # oldest first # Retrieve each sequentially. for num in changelists: args = [] if self.p4port: args.extend(['-p', self.p4port]) if self.p4user: args.extend(['-u', self.p4user]) if self.p4passwd: args.extend(['-P', self._getPasswd()]) args.extend(['describe', '-s', str(num)]) result = yield self._get_process_output(args) # decode the result from its designated encoding try: result = bytes2unicode(result, self.encoding) except UnicodeError as ex: log.msg( "P4Poller: couldn't decode changelist description: %s" % ex.encoding) log.msg("P4Poller: in object: %s" % ex.object) log.err("P4Poller: poll failed on %s, %s" % (self.p4port, self.p4base)) raise lines = result.split('\n') # SF#1555985: Wade Brainerd reports a stray ^M at the end of the date # field. The rstrip() is intended to remove that. lines[0] = lines[0].rstrip() m = self.describe_header_re.match(lines[0]) if not m: raise P4PollerError( "Unexpected 'p4 describe -s' result: %r" % result) who = self.resolvewho_callable(m.group('who')) when = datetime.datetime.strptime(m.group('when'), self.datefmt) if self.server_tz: # Convert from the server's timezone to the local timezone. when = when.replace(tzinfo=self.server_tz) when = util.datetime2epoch(when) comment_lines = [] lines.pop(0) # describe header lines.pop(0) # blank line while not lines[0].startswith('Affected files'): if lines[0].startswith('\t'): # comment is indented by one tab comment_lines.append(lines.pop(0)[1:]) else: lines.pop(0) # discard non comment line comments = '\n'.join(comment_lines) lines.pop(0) # affected files branch_files = {} # dict for branch mapped to file(s) while lines: line = lines.pop(0).strip() if not line: continue m = self.file_re.match(line) if not m: raise P4PollerError("Invalid file line: %r" % line) path = m.group('path') if path.startswith(self.p4base): branch, file = self.split_file(path[len(self.p4base):]) if (branch is None and file is None): continue if branch in branch_files: branch_files[branch].append(file) else: branch_files[branch] = [file] for branch in branch_files: yield self.master.data.updates.addChange( author=who, files=branch_files[branch], comments=comments, revision=text_type(num), when_timestamp=when, branch=branch, project=self.project, revlink=self.revlink_callable(branch, text_type(num))) self.last_change = num
def test_None(self): self.assertEqual(util.bytes2unicode(None, encoding='ascii'), None)
def addHTMLLog(name, html): _log = logfile.FakeLogFile(name) html = bytes2unicode(html) _log.addStdout(html) return defer.succeed(None)
def _redirect(master, request): url = request.args.get(b"redirect", [b"/"])[0] url = bytes2unicode(url) return resource.Redirect(master.config.buildbotURL + "#" + url)
def renderLogin(self, request): session = request.getSession() session.user_info = dict(username=bytes2unicode(self.username)) yield self.master.www.auth.updateUserInfo(request) raise _redirect(self.master, request)
def test_ascii(self): rv = util.bytes2unicode(b'abcd', encoding='ascii') self.assertEqual((rv, type(rv)), (u'abcd', text_type))