def assertReceivesChangeNewMessage(self, request): self.master.mq.callConsumer(("changes", "500", "new"), test_data_changes.Change.changeEvent) kw = self.readEvent(request) self.assertEqual(kw["event"], "event") msg = json.loads(kw["data"]) self.assertEqual(msg["key"], [u'changes', u'500', u'new']) self.assertEqual(msg["message"], json.loads(json.dumps(test_data_changes.Change.changeEvent, default=self._toJson)))
def _processChanges(self, page): result = json.loads(page, encoding=self.encoding) for pr in result["values"]: branch = pr["source"]["branch"]["name"] nr = int(pr["id"]) # Note that this is a short hash. The full length hash can be accessed via the # commit api resource but we want to avoid requesting multiple pages as long as # we are not sure that the pull request is new or updated. revision = pr["source"]["commit"]["hash"] # check branch if not self.branch or branch in self.branch: current = yield self._getCurrentRev(nr) if not current or current != revision: # parse pull request api page (required for the filter) page = yield client.getPage(str(pr["links"]["self"]["href"])) pr_json = json.loads(page, encoding=self.encoding) # filter pull requests by user function if not self.pullrequest_filter(pr_json): log.msg("pull request does not match filter") continue # access additional information author = pr["author"]["display_name"] prlink = pr["links"]["html"]["href"] # Get time updated time. Note that the timezone offset is ignored. if self.useTimestamps: updated = datetime.strptime(pr["updated_on"].split(".")[0], "%Y-%m-%dT%H:%M:%S") else: updated = epoch2datetime(reactor.seconds()) title = pr["title"] # parse commit api page page = yield client.getPage(str(pr["source"]["commit"]["links"]["self"]["href"])) commit_json = json.loads(page, encoding=self.encoding) # use the full-length hash from now on revision = commit_json["hash"] revlink = commit_json["links"]["html"]["href"] # parse repo api page page = yield client.getPage(str(pr["source"]["repository"]["links"]["self"]["href"])) repo_json = json.loads(page, encoding=self.encoding) repo = repo_json["links"]["html"]["href"] # update database yield self._setCurrentRev(nr, revision) # emit the change yield self.master.data.updates.addChange( author=ascii2unicode(author), revision=ascii2unicode(revision), revlink=ascii2unicode(revlink), comments=u"pull-request #%d: %s\n%s" % (nr, title, prlink), when_timestamp=datetime2epoch(updated), branch=self.branch, category=self.category, project=self.project, repository=ascii2unicode(repo), src=u"bitbucket", )
def getChanges(request, options=None): """ Consumes a naive build notification (the default for now) basically, set POST variables to match commit object parameters: revision, revlink, comments, branch, who, files, links files, links and properties will be de-json'd, the rest are interpreted as strings """ def firstOrNothing( value ): """ Small helper function to return the first value (if value is a list) or return the whole thing otherwise """ if ( type(value) == type([])): return value[0] else: return value args = request.args # first, convert files, links and properties files = None if args.get('files'): files = json.loads( args.get('files')[0] ) else: files = [] links = None if args.get('links'): links = json.loads( args.get('links')[0] ) else: links = [] properties = None if args.get('properties'): properties = json.loads( args.get('properties')[0] ) else: properties = {} revision = firstOrNothing(args.get('revision')) when = firstOrNothing(args.get('when')) who = firstOrNothing(args.get('who')) comments = firstOrNothing(args.get('comments')) isdir = firstOrNothing(args.get('isdir',0)) branch = firstOrNothing(args.get('branch')) category = firstOrNothing(args.get('category')) revlink = firstOrNothing(args.get('revlink')) repository = firstOrNothing(args.get('repository')) project = firstOrNothing(args.get('project')) chdict = dict(who=who, files=files, comments=comments, isdir=isdir, links=links, revision=revision, when=when, branch=branch, category=category, revlink=revlink, properties=properties, repository=repository, project=project) return ([ chdict ], None)
def _get_payload(request): content = request.content.read() content_type = request.getHeader(_HEADER_CT) if content_type == 'application/json': payload = json.loads(content) elif content_type == 'application/x-www-form-urlencoded': payload = json.loads(request.args['payload'][0]) else: raise ValueError('Unknown content type: %r' % (content_type,)) log.msg("Payload: %r" % payload, logLevel=logging.DEBUG) return payload
def testDefaultDialectWithChange(self): self.request.uri = "/change_hook/" ret = self.changeHook.render_GET(self.request) # Change is an array of dicts holding changes. There will normally only be one # changes, thus only one dictionary changeArray = json.loads(ret) change = changeArray[0] self.assertEquals(change["category"], "mycat") files = change["files"] self.assertEquals(len(files), 2) self.assertEquals(files[0]["name"], "file1") self.assertEquals(files[1]["name"], "file2") self.assertEquals(change["repository"], "myrepo") self.assertEquals(change["when"], 1234) self.assertEquals(change["who"], "Santa Claus") self.assertEquals(change["rev"], "99") self.assertEquals(change["number"], None) self.assertEquals(change["comments"], "a comment") self.assertEquals(change["project"], "a project") self.assertNotEquals(change["at"], "sometime") self.assertEquals(change["branch"], "a branch") self.assertEquals(change["revlink"], "a revlink") properties = change["properties"] self.assertEquals(len(properties), 2) self.assertEquals(properties[0][0], "prop1") self.assertEquals(properties[0][1], "val1") self.assertEquals(properties[0][2], "Change") self.assertEquals(properties[1][0], "prop2") self.assertEquals(properties[1][1], "val2") self.assertEquals(properties[1][2], "Change") self.assertEquals(change["revision"], "99")
def _chdict_from_change_row_thd(self, conn, ch_row): # This method must be run in a db.pool thread, and returns a chdict # given a row from the 'changes' table change_links_tbl = self.db.model.change_links change_files_tbl = self.db.model.change_files change_properties_tbl = self.db.model.change_properties def mkdt(epoch): if epoch: return epoch2datetime(epoch) chdict = ChDict( changeid=ch_row.changeid, author=ch_row.author, files=[], # see below comments=ch_row.comments, is_dir=ch_row.is_dir, links=[], # see below revision=ch_row.revision, when_timestamp=mkdt(ch_row.when_timestamp), branch=ch_row.branch, category=ch_row.category, revlink=ch_row.revlink, properties={}, # see below repository=ch_row.repository, project=ch_row.project, ) query = change_links_tbl.select(whereclause=(change_links_tbl.c.changeid == ch_row.changeid)) rows = conn.execute(query) for r in rows: chdict["links"].append(r.link) query = change_files_tbl.select(whereclause=(change_files_tbl.c.changeid == ch_row.changeid)) rows = conn.execute(query) for r in rows: chdict["files"].append(r.filename) # and properties must be given without a source, so strip that, but # be flexible in case users have used a development version where the # change properties were recorded incorrectly def split_vs(vs): try: v, s = vs if s != "Change": v, s = vs, "Change" except: v, s = vs, "Change" return v, s query = change_properties_tbl.select(whereclause=(change_properties_tbl.c.changeid == ch_row.changeid)) rows = conn.execute(query) for r in rows: try: v, s = split_vs(json.loads(r.property_value)) chdict["properties"][r.property_name] = (v, s) except ValueError: pass return chdict
def dataReceived(self, frame): log.msg("FRAME %s" % frame) # parse the incoming request # TODO: error handling frame = json.loads(frame) req = frame['req'] if req == 'startConsuming': path = tuple(frame['path']) options = frame['options'] # if it's already subscribed, don't leak a subscription if path in self.qrefs: return def callback(key, message): content = json.dumps(dict(path=path, key=key, message=message)) self.transport.write(content) d = self.master.data.startConsuming(callback, options, path) @d.addCallback def register(qref): if path in self.qrefs: qref.stopConsuming() self.qrefs[path] = qref d.addErrback("while starting consumption")
def insertTestData(self, rows): for row in rows: if isinstance(row, Change): ch = self.ChangeInstance() ch.__dict__.update(dict( # TODO: this renaming sucks. number=row.changeid, who=row.author, files=[], comments=row.comments, isdir=row.is_dir, links=[], revision=row.revision, when=row.when_timestamp, branch=row.branch, category=row.category, revlink=row.revlink, properties={}, repository=row.repository, project=row.project)) self.changes[row.changeid] = ch elif isinstance(row, ChangeFile): ch = self.changes[row.changeid] ch.files.append(row.filename) elif isinstance(row, ChangeLink): ch = self.changes[row.changeid] ch.links.append(row.link) elif isinstance(row, ChangeProperty): ch = self.changes[row.changeid] n, v = row.property_name, row.property_value ch.properties[n] = json.loads(v)
def _check(page): data = json.loads(page) self.assertEqual(len(data), 4) self.assertEqual(len(data["builders"]), 1) self.assertEqual(len(data["change_sources"]), 1) self.assertEqual(len(data["project"]), 2) self.assertEqual(len(data["slaves"]), 1)
def thd( ): # everything in deferToThread is not counted with trial --coverage :-( url = self.tokenUri data = { 'redirect_uri': self.loginUri, 'code': code, 'grant_type': self.grantType } auth = None if self.getTokenUseAuthHeaders: auth = (self.clientId, self.clientSecret) else: data.update({ 'client_id': self.clientId, 'client_secret': self.clientSecret }) data.update(self.tokenUriAdditionalParams) response = requests.post(url, data=data, auth=auth, verify=self.sslVerify) response.raise_for_status() if isinstance(response.content, basestring): try: content = json.loads(response.content) except ValueError: content = parse_qs(response.content) for k, v in iteritems(content): content[k] = v[0] else: content = response.content session = self.createSessionFromToken(content) return self.getUserInfoFromOAuthClient(session)
def assertStateByClass(self, name, class_name, **kwargs): objectid = self.objects[(name, class_name)] state = self.states[objectid] for k, v in kwargs.iteritems(): self.t.assertIn(k, state) self.t.assertEqual(json.loads(state[k]), v, "state is %r" % (state,))
def _stepdictFromRow(self, row): def mkdt(epoch): if epoch: return epoch2datetime(epoch) return dict( id=row.id, number=row.number, name=row.name, buildid=row.buildid, started_at=mkdt(row.started_at), complete_at=mkdt(row.complete_at), state_strings=json.loads(row.state_strings_json), results=row.results, urls=json.loads(row.urls_json), )
def insertTestData(self, rows): for row in rows: if isinstance(row, Change): ch = self.ChangeInstance() ch.__dict__.update(dict( # TODO: this renaming sucks. number=row.changeid, who=row.author, files=[], comments=row.comments, isdir=row.is_dir, links=[], revision=row.revision, when=row.when_timestamp, branch=row.branch, category=row.category, revlink=row.revlink, properties=properties.Properties(), repository=row.repository, project=row.project, uid=None)) self.changes[row.changeid] = ch elif isinstance(row, ChangeFile): ch = self.changes[row.changeid] ch.files.append(row.filename) elif isinstance(row, ChangeLink): ch = self.changes[row.changeid] ch.links.append(row.link) elif isinstance(row, ChangeProperty): ch = self.changes[row.changeid] n, vs = row.property_name, row.property_value v, s = json.loads(vs) ch.properties.setProperty(n, v, s) elif isinstance(row, ChangeUser): ch = self.changes[row.changeid] ch.uid = row.uid
def lineReceived(self, line): try: event = json.loads(line.decode('utf-8')) except ValueError: log.msg("bad json line: %s" % (line,)) return defer.succeed(None) if not(type(event) == type({}) and "type" in event): log.msg("no type in event %s" % (line,)) return defer.succeed(None) func = getattr(self, "eventReceived_"+event["type"].replace("-","_"), None) if func == None: log.msg("unsupported event %s" % (event["type"],)) return defer.succeed(None) # flatten the event dictionary, for easy access with WithProperties def flatten(properties, base, event): for k, v in event.items(): if type(v) == dict: flatten(properties, base + "." + k, v) else: # already there properties[base + "." + k] = v properties = {} flatten(properties, "event", event) return func(properties,event)
def thd(): url = self.tokenUri data = {'redirect_uri': self.loginUri, 'code': code, 'grant_type': self.grantType} auth = None if self.getTokenUseAuthHeaders: auth = (self.clientId, self.clientSecret) else: data.update( {'client_id': self.clientId, 'client_secret': self.clientSecret}) data.update(self.tokenUriAdditionalParams) response = requests.post( url, data=data, auth=auth, verify=self.sslVerify) response.raise_for_status() if isinstance(response.content, basestring): try: content = json.loads(response.content) except ValueError: content = parse_qs(response.content) for k, v in iteritems(content): content[k] = v[0] else: content = response.content session = self.createSessionFromToken(content) return self.getUserInfoFromOAuthClient(session)
def decodeJsonRPC2(self, request): # Verify the content-type. Browsers are easily convinced to send # POST data to arbitrary URLs via 'form' elements, but they won't # use the application/json content-type. if ContentTypeParser(request.getHeader("content-type")).gettype() != JSON_ENCODED: raise BadJsonRpc2("Invalid content-type (use application/json)", JSONRPC_CODES["invalid_request"]) try: data = json.loads(request.content.read()) except Exception as e: raise BadJsonRpc2("JSON parse error: %s" % (str(e),), JSONRPC_CODES["parse_error"]) if isinstance(data, list): raise BadJsonRpc2("JSONRPC batch requests are not supported", JSONRPC_CODES["invalid_request"]) if not isinstance(data, dict): raise BadJsonRpc2("JSONRPC root object must be an object", JSONRPC_CODES["invalid_request"]) def check(name, types, typename): if name not in data: raise BadJsonRpc2("missing key '%s'" % (name,), JSONRPC_CODES["invalid_request"]) if not isinstance(data[name], types): raise BadJsonRpc2("'%s' must be %s" % (name, typename), JSONRPC_CODES["invalid_request"]) check("jsonrpc", (str, unicode), "a string") check("method", (str, unicode), "a string") check("id", (str, unicode, int, type(None)), "a string, number, or null") check("params", (dict,), "an object") if data["jsonrpc"] != "2.0": raise BadJsonRpc2("only JSONRPC 2.0 is supported", JSONRPC_CODES["invalid_request"]) return data["method"], data["id"], data["params"]
def assertRestCollection(self, typeName, items, total=None, contentType=None, orderSignificant=False): self.failIf(isinstance(self.request.written, unicode)) got = {} got['content'] = json.loads(self.request.written) got['contentType'] = self.request.headers['content-type'] got['responseCode'] = self.request.responseCode meta = {} if total is not None: meta['total'] = total exp = {} exp['content'] = {typeName: items, 'meta': meta} exp['contentType'] = [contentType or 'text/plain; charset=utf-8'] exp['responseCode'] = 200 # if order is not significant, sort so the comparison works if not orderSignificant: if 'content' in got and typeName in got['content']: got['content'][typeName].sort() exp['content'][typeName].sort() if 'meta' in got['content'] and 'links' in got['content']['meta']: got['content']['meta']['links'].sort( key=lambda l: (l['rel'], l['href'])) self.assertEqual(got, exp)
def lineReceived(self, line): try: event = json.loads(line.decode('utf-8')) except ValueError: log.msg("bad json line: %s" % (line,)) return defer.succeed(None) if not(type(event) == type({}) and "type" in event): log.msg("no type in event %s" % (line,)) return defer.succeed(None) func = getattr(self, "eventReceived_"+event["type"].replace("-","_"), None) if func == None: log.msg("unsupported event %s" % (event["type"],)) return defer.succeed(None) # flatten the event dictionary, for easy access with WithProperties def flatten(event, base, d): for k, v in d.items(): if type(v) == dict: flatten(event, base + "." + k, v) else: # already there event[base + "." + k] = v properties = {} flatten(properties, "event", event) return func(properties,event)
def assertRestCollection(self, typeName, items, total=None, contentType=None, orderSignificant=False): self.failIf(isinstance(self.request.written, unicode)) got = {} got["content"] = json.loads(self.request.written) got["contentType"] = self.request.headers["content-type"] got["responseCode"] = self.request.responseCode meta = {} if total is not None: meta["total"] = total exp = {} exp["content"] = {typeName: items, "meta": meta} exp["contentType"] = [contentType or "text/plain; charset=utf-8"] exp["responseCode"] = 200 # if order is not significant, sort so the comparison works if not orderSignificant: if "content" in got and typeName in got["content"]: got["content"][typeName].sort() exp["content"][typeName].sort() if "meta" in got["content"] and "links" in got["content"]["meta"]: got["content"]["meta"]["links"].sort(key=lambda l: (l["rel"], l["href"])) self.assertEqual(got, exp)
def testDefaultDialectWithChange(self): self.request.uri = "/change_hook/" ret = self.changeHook.render_GET(self.request) # Change is an array of dicts holding changes. There will normally only be one # changes, thus only one dictionary changeArray = json.loads(ret) change = changeArray[0] self.assertEquals(change["category"], "mycat") files = change["files"] self.assertEquals(len(files), 2) self.assertEquals(files[0]["name"], "file1") self.assertEquals(files[1]["name"], "file2") self.assertEquals(change["repository"], "myrepo") self.assertEquals(change["when"], 1234) self.assertEquals(change["who"], "Santa Claus") self.assertEquals(change["rev"], '99') self.assertEquals(change["number"], None) self.assertEquals(change["comments"], "a comment") self.assertEquals(change["project"], "a project") self.assertNotEquals(change["at"], "sometime") self.assertEquals(change["branch"], "a branch") self.assertEquals(change["revlink"], "a revlink") properties = change["properties"] self.assertEquals(len(properties), 2) self.assertEquals(properties[0][0], "prop1") self.assertEquals(properties[0][1], "val1") self.assertEquals(properties[0][2], "Change") self.assertEquals(properties[1][0], "prop2") self.assertEquals(properties[1][1], "val2") self.assertEquals(properties[1][2], "Change") self.assertEquals(change["revision"], '99')
def assertStateByClass(self, name, class_name, **kwargs): objectid = self.objects[(name, class_name)] state = self.states[objectid] for k, v in kwargs.iteritems(): self.t.assertIn(k, state) self.t.assertEqual(json.loads(state[k]), v, "state is %r" % (state, ))
def thd(conn): bsp_tbl = self.db.model.buildset_properties q = sa.select([bsp_tbl.c.property_name, bsp_tbl.c.property_value], whereclause=(bsp_tbl.c.buildsetid == buildsetid)) return dict([(row.property_name, tuple(json.loads(row.property_value))) for row in conn.execute(q)])
def onMessage(self, frame, isBinary): log.msg("FRAME %s" % frame) # parse the incoming request frame = json.loads(frame) _id = frame.get("_id") if _id is None: return self.sendJsonMessage(error="no '_id' in websocket frame", code=400, _id=None) cmd = frame.pop("cmd", None) if cmd is None: return self.sendJsonMessage(error="no 'cmd' in websocket frame", code=400, _id=None) cmdmeth = "cmd_" + cmd meth = getattr(self, cmdmeth, None) if meth is None: return self.sendJsonMessage(error="no such command '%s'" % (cmd, ), code=404, _id=_id) try: return meth(**frame) except TypeError as e: return self.sendJsonMessage(error="Invalid method argument '%s'" % (str(e), ), code=400, _id=_id) except Exception as e: log.err("while calling command %s" % (cmd, )) return self.sendJsonMessage(error="Internal Error '%s'" % (str(e), ), code=500, _id=_id)
def _chdict_from_change_row_thd(self, conn, ch_row): # This method must be run in a db.pool thread, and returns a chdict # given a row from the 'changes' table change_links_tbl = self.db.model.change_links change_files_tbl = self.db.model.change_files change_properties_tbl = self.db.model.change_properties def mkdt(epoch): if epoch: return epoch2datetime(epoch) chdict = ChDict( changeid=ch_row.changeid, author=ch_row.author, files=[], # see below comments=ch_row.comments, is_dir=ch_row.is_dir, links=[], # see below revision=ch_row.revision, when_timestamp=mkdt(ch_row.when_timestamp), branch=ch_row.branch, category=ch_row.category, revlink=ch_row.revlink, properties={}, # see below repository=ch_row.repository, project=ch_row.project) query = change_links_tbl.select( whereclause=(change_links_tbl.c.changeid == ch_row.changeid)) rows = conn.execute(query) for r in rows: chdict['links'].append(r.link) query = change_files_tbl.select( whereclause=(change_files_tbl.c.changeid == ch_row.changeid)) rows = conn.execute(query) for r in rows: chdict['files'].append(r.filename) # and properties must be given without a source, so strip that, but # be flexible in case users have used a development version where the # change properties were recorded incorrectly def split_vs(vs): try: v, s = vs if s != "Change": v, s = vs, "Change" except: v, s = vs, "Change" return v, s query = change_properties_tbl.select( whereclause=(change_properties_tbl.c.changeid == ch_row.changeid)) rows = conn.execute(query) for r in rows: v, s = split_vs(json.loads(r.property_value)) chdict['properties'][r.property_name] = (v, s) return chdict
def assertState(self, objectid, missing_keys=[], **kwargs): state = self.states[objectid] for k in missing_keys: self.t.assertFalse(k in state, "%s in %s" % (k, state)) for k, v in kwargs.iteritems(): self.t.assertIn(k, state) self.t.assertEqual(json.loads(state[k]), v, "state is %r" % (state,))
def assertState(self, objectid, missing_keys=[], **kwargs): state = self.states[objectid] for k in missing_keys: self.t.assertFalse(k in state, "%s in %s" % (k, state)) for k, v in kwargs.iteritems(): self.t.assertIn(k, state) self.t.assertEqual(json.loads(state[k]), v, "state is %r" % (state, ))
def insertTestData(self, rows): for row in rows: if isinstance(row, Scheduler): self.states[row.schedulerid] = json.loads(row.state) elif isinstance(row, SchedulerChange): cls = self.classifications.setdefault(row.schedulerid, {}) cls[row.changeid] = row.important
def validate(self, name, object): if not isinstance(object, dict): # we want a dict, and NOT a subclass yield "%s is not sourced properties (not a dict)" % (name, ) return for k, v in object.iteritems(): if not isinstance(k, unicode): yield "%s property name %r is not unicode" % (name, k) if not isinstance(v, tuple) or len(v) != 2: yield "%s property value for '%s' is not a 2-tuple" % (name, k) return propval, propsrc = v if not isinstance(propsrc, unicode): yield "%s[%s] source %r is not unicode" % (name, k, propsrc) try: json.loads(propval) except ValueError: yield "%s[%r] value is not JSON-able" % (name, k)
def getState(self, objectid, name, default=object): try: json_value = self.states[objectid][name] except KeyError: if default is not object: return defer.succeed(default) raise return defer.succeed(json.loads(json_value))
def check(_json): res = json.loads(_json) self.assertIn("jsonrpc", res) self.assertEqual(res["jsonrpc"], "2.0") if not requestJson: # requestJson is used for invalid requests, so don't expect ID self.assertIn("id", res) self.assertEqual(res["id"], id)
def thd(conn): bsp_tbl = self.db.model.buildset_properties q = sa.select( [ bsp_tbl.c.property_name, bsp_tbl.c.property_value ], whereclause=(bsp_tbl.c.buildsetid == buildsetid)) return dict([ (row.property_name, tuple(json.loads(row.property_value))) for row in conn.execute(q) ])
def validate(self, name, object): if not isinstance(object, dict): # we want a dict, and NOT a subclass yield "%s is not sourced properties (not a dict)" % (name,) return for k, v in iteritems(object): if not isinstance(k, unicode): yield "%s property name %r is not unicode" % (name, k) if not isinstance(v, tuple) or len(v) != 2: yield "%s property value for '%s' is not a 2-tuple" % (name, k) return propval, propsrc = v if not isinstance(propsrc, unicode): yield "%s[%s] source %r is not unicode" % (name, k, propsrc) try: json.loads(propval) except ValueError: yield "%s[%r] value is not JSON-able" % (name, k)
def decodeJsonRPC2(self, request): # Content-Type is ignored, so that AJAX requests can be sent without # incurring CORS preflight overheads. The JSONRPC spec does not # suggest a Content-Type anyway. try: data = json.loads(request.content.read()) except Exception, e: raise BadJsonRpc2("JSON parse error: %s" % (str(e), ), JSONRPC_CODES["parse_error"])
def decodeJsonRPC2(self, request): # Content-Type is ignored, so that AJAX requests can be sent without # incurring CORS preflight overheads. The JSONRPC spec does not # suggest a Content-Type anyway. try: data = json.loads(request.content.read()) except Exception, e: raise BadJsonRpc2("JSON parse error: %s" % (str(e),), JSONRPC_CODES["parse_error"])
def thd(conn): bp_tbl = self.db.model.build_properties q = sa.select([bp_tbl.c.name, bp_tbl.c.value, bp_tbl.c.source], whereclause=(bp_tbl.c.buildid == bid)) props = [] for row in conn.execute(q): prop = (json.loads(row.value), row.source) props.append((row.name, prop)) return dict(props)
def lineReceived(self, line): try: event = json.loads(line.decode('utf-8')) except ValueError: log.msg("bad json line: %s" % (line,)) return defer.succeed(None) if type(event) == type({}) and "type" in event and event["type"] in ["patchset-created", "ref-updated"]: # flatten the event dictionary, for easy access with WithProperties def flatten(event, base, d): for k, v in d.items(): if type(v) == dict: flatten(event, base + "." + k, v) else: # already there event[base + "." + k] = v properties = {} flatten(properties, "event", event) if event["type"] == "patchset-created": change = event["change"] chdict = dict( author="%s <%s>" % (change["owner"]["name"], change["owner"]["email"]), project=change["project"], branch=change["branch"], revision=event["patchSet"]["revision"], revlink=change["url"], comments=change["subject"], files=["unknown"], category=event["type"], properties=properties) elif event["type"] == "ref-updated": ref = event["refUpdate"] author = "gerrit" if "submitter" in event: author="%s <%s>" % (event["submitter"]["name"], event["submitter"]["email"]) chdict = dict( author=author, project=ref["project"], branch=ref["refName"], revision=ref["newRev"], comments="Gerrit: patchset(s) merged.", files=["unknown"], category=event["type"], properties=properties) else: return defer.succeed(None) # this shouldn't happen anyway d = self.master.addChange(**chdict) # eat failures.. d.addErrback(log.err, 'error adding change from GerritChangeSource') return d else: return defer.succeed(None)
def getJSON(data): push = json.loads(data) log.msg("Looking at the push json data for try comments") for p in push: pd = push[p] changes = pd['changesets'] for change in reversed(changes): match = re.search("try:", change['desc']) if match: return change['desc'].encode("utf8", "replace")
def __init__(self, headers, body, branch): self._auth_code = headers['Google-Code-Project-Hosting-Hook-Hmac'] self._body = body # we need to save it if we want to authenticate it self._branch = branch payload = json.loads(body) self.project = payload['project_name'] self.repository = payload['repository_path'] self.revisions = payload['revisions'] self.revision_count = payload['revision_count']
def assertRestError(self, responseCode, message): got = {} got['content'] = json.loads(self.request.written) got['responseCode'] = self.request.responseCode exp = {} exp['content'] = {'error': message} exp['responseCode'] = responseCode self.assertEqual(got, exp)
def assertRestError(self, responseCode, message): got = {} got["content"] = json.loads(self.request.written) got["responseCode"] = self.request.responseCode exp = {} exp["content"] = {"error": message} exp["responseCode"] = responseCode self.assertEqual(got, exp)
def thd(conn): bp_tbl = self.db.model.build_properties q = sa.select( [bp_tbl.c.name, bp_tbl.c.value, bp_tbl.c.source], whereclause=(bp_tbl.c.buildid == bid)) props = [] for row in conn.execute(q): prop = (json.loads(row.value), row.source) props.append((row.name, prop)) return dict(props)
def _txn_get_properties_from_db(self, t, tablename, idname, id): # apparently you can't use argument placeholders for table names. Don't # call this with a weird-looking tablename. q = self.quoteq("SELECT property_name,property_value FROM %s WHERE %s=?" % (tablename, idname)) t.execute(q, (id,)) retval = Properties() for key, valuepair in t.fetchall(): value, source = json.loads(valuepair) retval.setProperty(str(key), value, source) return retval
def getJSON(data): push = json.loads(data) log.msg("Looking at the push json data for try comments") for p in push: pd = push[p] changes = pd["changesets"] for change in reversed(changes): match = re.search("try:", change["desc"]) if match: return change["desc"].encode("utf8", "replace")
def lineReceived(self, line): try: event = json.loads(line) except ValueError: log.msg("bad json line: %s" % (line, )) return defer.succeed(None) if type(event) == type({}) and "type" in event and event["type"] in [ "patchset-created", "ref-updated" ]: # flatten the event dictionary, for easy access with WithProperties def flatten(event, base, d): for k, v in d.items(): if type(v) == dict: flatten(event, base + "." + k, v) else: # already there event[base + "." + k] = v properties = {} flatten(properties, "event", event) if event["type"] == "patchset-created": change = event["change"] chdict = dict( who="%s <%s>" % (change["owner"]["name"], change["owner"]["email"]), project=change["project"], branch=change["branch"], revision=event["patchSet"]["revision"], revlink=change["url"], comments=change["subject"], files=["unknown"], category=event["type"], properties=properties) elif event["type"] == "ref-updated": ref = event["refUpdate"] chdict = dict( who="%s <%s>" % (event["submitter"]["name"], event["submitter"]["email"]), project=ref["project"], branch=ref["refName"], revision=ref["newRev"], comments="Gerrit: patchset(s) merged.", files=["unknown"], category=event["type"], properties=properties) else: return defer.succeed(None) # this shouldn't happen anyway d = self.master.addChange(**chdict) # eat failures.. d.addErrback(log.err, 'error adding change from GerritChangeSource') return d
def testGitWithChange(self): self.request.uri = "/change_hook/github" ret = self.changeHook.render_GET(self.request) # Change is an array of dicts holding changes. There may be multiple entries for github changeArray = json.loads(ret) change = changeArray[0] self.assertEquals(change["category"], None) files = change["files"] self.assertEquals(len(files), 1) self.assertEquals(files[0]["name"], "filepath.rb") self.assertEquals(change["repository"], "http://github.com/defunkt/github") self.assertEquals(change["when"], 1203116237) self.assertEquals(change["who"], "Fred Flinstone <*****@*****.**>") self.assertEquals(change["rev"], '41a212ee83ca127e3c8cf465891ab7216a705f59') self.assertEquals(change["number"], None) self.assertEquals(change["comments"], "okay i give in") self.assertEquals(change["project"], '') self.assertNotEquals(change["at"], "sometime") self.assertEquals(change["branch"], "master") self.assertEquals( change["revlink"], "http://github.com/defunkt/github/commit/41a212ee83ca127e3c8cf465891ab7216a705f59" ) properties = change["properties"] self.assertEquals(len(properties), 0) self.assertEquals(change["revision"], '41a212ee83ca127e3c8cf465891ab7216a705f59') # Second change change = changeArray[1] self.assertEquals(change["category"], None) files = change["files"] self.assertEquals(len(files), 2) self.assertEquals(files[0]["name"], "modfile") self.assertEquals(files[1]["name"], "removedFile") self.assertEquals(change["repository"], "http://github.com/defunkt/github") self.assertEquals(change["when"], 1203114994) self.assertEquals(change["who"], "Fred Flinstone <*****@*****.**>") self.assertEquals(change["rev"], 'de8251ff97ee194a289832576287d6f8ad74e3d0') self.assertEquals(change["number"], None) self.assertEquals(change["comments"], "update pricing a tad") self.assertEquals(change["project"], '') self.assertNotEquals(change["at"], "sometime") self.assertEquals(change["branch"], "master") self.assertEquals( change["revlink"], "http://github.com/defunkt/github/commit/de8251ff97ee194a289832576287d6f8ad74e3d0" ) properties = change["properties"] self.assertEquals(len(properties), 0) self.assertEquals(change["revision"], 'de8251ff97ee194a289832576287d6f8ad74e3d0')
def thd(conn): schedulers_tbl = self.db.model.schedulers q = sa.select([ schedulers_tbl.c.state ], whereclause=(schedulers_tbl.c.schedulerid == schedulerid)) row = conn.execute(q).fetchone() if not row: return {} # really shouldn't happen - the row should exist try: return json.loads(row.state) except: log.msg("JSON error loading state for scheduler #%s" % (schedulerid,)) return {}
def _txn_get_properties_from_db(self, t, tablename, idname, id): # apparently you can't use argument placeholders for table names. Don't # call this with a weird-looking tablename. q = self.quoteq( "SELECT property_name,property_value FROM %s WHERE %s=?" % (tablename, idname)) t.execute(q, (id, )) retval = Properties() for key, valuepair in t.fetchall(): value, source = json.loads(valuepair) retval.setProperty(str(key), value, source) return retval
def thd(conn): bsp_tbl = self.db.model.buildset_properties q = sa.select([bsp_tbl.c.property_name, bsp_tbl.c.property_value], whereclause=(bsp_tbl.c.buildsetid == buildsetid)) l = [] for row in conn.execute(q): try: properties = json.loads(row.property_value) l.append((row.property_name, tuple(properties))) except ValueError: pass return dict(l)
def insertTestData(self, rows): for row in rows: if isinstance(row, Buildset): bs = self.buildsets[row.id] = row.values.copy() bs['properties'] = {} for row in rows: if isinstance(row, BuildsetProperty): assert row.buildsetid in self.buildsets n = row.property_name v, src = tuple(json.loads(row.property_value)) self.buildsets[row.buildsetid]['properties'][n] = (v, src)
def thd(conn): rows = self.getObjectStateData(conn, objects) try: object_state = {} for row in rows: object_state[row.name] = json.loads(row.value_json).keys() return object_state except: raise TypeError("JSON error loading state value '%s'" % (objects))
def decodeJsonRPC2(self, request): # Verify the content-type. Browsers are easily convinced to send # POST data to arbitrary URLs via 'form' elements, but they won't # use the application/json content-type. if request.getHeader('content-type') != 'application/json': raise BadJsonRpc2('Invalid content-type (use application/json)', JSONRPC_CODES["invalid_request"]) try: data = json.loads(request.content.read()) except Exception, e: raise BadJsonRpc2("JSON parse error: %s" % (str(e), ), JSONRPC_CODES["parse_error"])
def thd(conn): tbl = self.db.model.steps wc = (tbl.c.id == stepid) q = sa.select([tbl.c.urls_json], whereclause=wc) res = conn.execute(q) row = res.fetchone() if _racehook is not None: _racehook() urls = json.loads(row.urls_json) urls.append(dict(name=name, url=url)) q = tbl.update(whereclause=wc) conn.execute(q, urls_json=json.dumps(urls))
def insertTestData(self, rows): for row in rows: if isinstance(row, Buildset): bs = self.buildsets[row.id] = row.values.copy() bs['properties'] = {} if isinstance(row, SchedulerUpstreamBuildset): self.buildset_subs.append((row.schedulerid, row.buildsetid)) for row in rows: if isinstance(row, BuildsetProperty): assert row.buildsetid in self.buildsets n = row.property_name v, src = tuple(json.loads(row.property_value)) self.buildsets[row.buildsetid]['properties'][n] = (v, src)