def assertReceivesChangeNewMessage(self, request):
self.master.mq.callConsumer(("changes", "500", "new"), test_data_changes.Change.changeEvent)
kw = self.readEvent(request)
self.assertEqual(kw["event"], "event")
msg = json.loads(kw["data"])
self.assertEqual(msg["key"], [u'changes', u'500', u'new'])
self.assertEqual(msg["message"], json.loads(json.dumps(test_data_changes.Change.changeEvent, default=self._toJson)))
def _processChanges(self, page):
result = json.loads(page, encoding=self.encoding)
for pr in result["values"]:
branch = pr["source"]["branch"]["name"]
nr = int(pr["id"])
# Note that this is a short hash. The full length hash can be accessed via the
# commit api resource but we want to avoid requesting multiple pages as long as
# we are not sure that the pull request is new or updated.
revision = pr["source"]["commit"]["hash"]
# check branch
if not self.branch or branch in self.branch:
current = yield self._getCurrentRev(nr)
if not current or current != revision:
# parse pull request api page (required for the filter)
page = yield client.getPage(str(pr["links"]["self"]["href"]))
pr_json = json.loads(page, encoding=self.encoding)
# filter pull requests by user function
if not self.pullrequest_filter(pr_json):
log.msg("pull request does not match filter")
continue
# access additional information
author = pr["author"]["display_name"]
prlink = pr["links"]["html"]["href"]
# Get time updated time. Note that the timezone offset is ignored.
if self.useTimestamps:
updated = datetime.strptime(pr["updated_on"].split(".")[0], "%Y-%m-%dT%H:%M:%S")
else:
updated = epoch2datetime(reactor.seconds())
title = pr["title"]
# parse commit api page
page = yield client.getPage(str(pr["source"]["commit"]["links"]["self"]["href"]))
commit_json = json.loads(page, encoding=self.encoding)
# use the full-length hash from now on
revision = commit_json["hash"]
revlink = commit_json["links"]["html"]["href"]
# parse repo api page
page = yield client.getPage(str(pr["source"]["repository"]["links"]["self"]["href"]))
repo_json = json.loads(page, encoding=self.encoding)
repo = repo_json["links"]["html"]["href"]
# update database
yield self._setCurrentRev(nr, revision)
# emit the change
yield self.master.data.updates.addChange(
author=ascii2unicode(author),
revision=ascii2unicode(revision),
revlink=ascii2unicode(revlink),
comments=u"pull-request #%d: %s\n%s" % (nr, title, prlink),
when_timestamp=datetime2epoch(updated),
branch=self.branch,
category=self.category,
project=self.project,
repository=ascii2unicode(repo),
src=u"bitbucket",
)
def getChanges(request, options=None):
"""
Consumes a naive build notification (the default for now)
basically, set POST variables to match commit object parameters:
revision, revlink, comments, branch, who, files, links
files, links and properties will be de-json'd, the rest are interpreted as strings
"""
def firstOrNothing( value ):
"""
Small helper function to return the first value (if value is a list)
or return the whole thing otherwise
"""
if ( type(value) == type([])):
return value[0]
else:
return value
args = request.args
# first, convert files, links and properties
files = None
if args.get('files'):
files = json.loads( args.get('files')[0] )
else:
files = []
links = None
if args.get('links'):
links = json.loads( args.get('links')[0] )
else:
links = []
properties = None
if args.get('properties'):
properties = json.loads( args.get('properties')[0] )
else:
properties = {}
revision = firstOrNothing(args.get('revision'))
when = firstOrNothing(args.get('when'))
who = firstOrNothing(args.get('who'))
comments = firstOrNothing(args.get('comments'))
isdir = firstOrNothing(args.get('isdir',0))
branch = firstOrNothing(args.get('branch'))
category = firstOrNothing(args.get('category'))
revlink = firstOrNothing(args.get('revlink'))
repository = firstOrNothing(args.get('repository'))
project = firstOrNothing(args.get('project'))
chdict = dict(who=who, files=files, comments=comments,
isdir=isdir, links=links, revision=revision, when=when,
branch=branch, category=category, revlink=revlink,
properties=properties, repository=repository,
project=project)
return ([ chdict ], None)
def _get_payload(request):
content = request.content.read()
content_type = request.getHeader(_HEADER_CT)
if content_type == 'application/json':
payload = json.loads(content)
elif content_type == 'application/x-www-form-urlencoded':
payload = json.loads(request.args['payload'][0])
else:
raise ValueError('Unknown content type: %r' % (content_type,))
log.msg("Payload: %r" % payload, logLevel=logging.DEBUG)
return payload
def testDefaultDialectWithChange(self):
self.request.uri = "/change_hook/"
ret = self.changeHook.render_GET(self.request)
# Change is an array of dicts holding changes. There will normally only be one
# changes, thus only one dictionary
changeArray = json.loads(ret)
change = changeArray[0]
self.assertEquals(change["category"], "mycat")
files = change["files"]
self.assertEquals(len(files), 2)
self.assertEquals(files[0]["name"], "file1")
self.assertEquals(files[1]["name"], "file2")
self.assertEquals(change["repository"], "myrepo")
self.assertEquals(change["when"], 1234)
self.assertEquals(change["who"], "Santa Claus")
self.assertEquals(change["rev"], "99")
self.assertEquals(change["number"], None)
self.assertEquals(change["comments"], "a comment")
self.assertEquals(change["project"], "a project")
self.assertNotEquals(change["at"], "sometime")
self.assertEquals(change["branch"], "a branch")
self.assertEquals(change["revlink"], "a revlink")
properties = change["properties"]
self.assertEquals(len(properties), 2)
self.assertEquals(properties[0][0], "prop1")
self.assertEquals(properties[0][1], "val1")
self.assertEquals(properties[0][2], "Change")
self.assertEquals(properties[1][0], "prop2")
self.assertEquals(properties[1][1], "val2")
self.assertEquals(properties[1][2], "Change")
self.assertEquals(change["revision"], "99")
def _chdict_from_change_row_thd(self, conn, ch_row):
# This method must be run in a db.pool thread, and returns a chdict
# given a row from the 'changes' table
change_links_tbl = self.db.model.change_links
change_files_tbl = self.db.model.change_files
change_properties_tbl = self.db.model.change_properties
def mkdt(epoch):
if epoch:
return epoch2datetime(epoch)
chdict = ChDict(
changeid=ch_row.changeid,
author=ch_row.author,
files=[], # see below
comments=ch_row.comments,
is_dir=ch_row.is_dir,
links=[], # see below
revision=ch_row.revision,
when_timestamp=mkdt(ch_row.when_timestamp),
branch=ch_row.branch,
category=ch_row.category,
revlink=ch_row.revlink,
properties={}, # see below
repository=ch_row.repository,
project=ch_row.project,
)
query = change_links_tbl.select(whereclause=(change_links_tbl.c.changeid == ch_row.changeid))
rows = conn.execute(query)
for r in rows:
chdict["links"].append(r.link)
query = change_files_tbl.select(whereclause=(change_files_tbl.c.changeid == ch_row.changeid))
rows = conn.execute(query)
for r in rows:
chdict["files"].append(r.filename)
# and properties must be given without a source, so strip that, but
# be flexible in case users have used a development version where the
# change properties were recorded incorrectly
def split_vs(vs):
try:
v, s = vs
if s != "Change":
v, s = vs, "Change"
except:
v, s = vs, "Change"
return v, s
query = change_properties_tbl.select(whereclause=(change_properties_tbl.c.changeid == ch_row.changeid))
rows = conn.execute(query)
for r in rows:
try:
v, s = split_vs(json.loads(r.property_value))
chdict["properties"][r.property_name] = (v, s)
except ValueError:
pass
return chdict
def dataReceived(self, frame):
log.msg("FRAME %s" % frame)
# parse the incoming request
# TODO: error handling
frame = json.loads(frame)
req = frame['req']
if req == 'startConsuming':
path = tuple(frame['path'])
options = frame['options']
# if it's already subscribed, don't leak a subscription
if path in self.qrefs:
return
def callback(key, message):
content = json.dumps(dict(path=path, key=key, message=message))
self.transport.write(content)
d = self.master.data.startConsuming(callback, options, path)
@d.addCallback
def register(qref):
if path in self.qrefs:
qref.stopConsuming()
self.qrefs[path] = qref
d.addErrback("while starting consumption")
def insertTestData(self, rows):
for row in rows:
if isinstance(row, Change):
ch = self.ChangeInstance()
ch.__dict__.update(dict(
# TODO: this renaming sucks.
number=row.changeid, who=row.author, files=[],
comments=row.comments, isdir=row.is_dir, links=[],
revision=row.revision, when=row.when_timestamp,
branch=row.branch, category=row.category,
revlink=row.revlink, properties={},
repository=row.repository, project=row.project))
self.changes[row.changeid] = ch
elif isinstance(row, ChangeFile):
ch = self.changes[row.changeid]
ch.files.append(row.filename)
elif isinstance(row, ChangeLink):
ch = self.changes[row.changeid]
ch.links.append(row.link)
elif isinstance(row, ChangeProperty):
ch = self.changes[row.changeid]
n, v = row.property_name, row.property_value
ch.properties[n] = json.loads(v)
def _check(page):
data = json.loads(page)
self.assertEqual(len(data), 4)
self.assertEqual(len(data["builders"]), 1)
self.assertEqual(len(data["change_sources"]), 1)
self.assertEqual(len(data["project"]), 2)
self.assertEqual(len(data["slaves"]), 1)
def dataReceived(self, frame):
log.msg("FRAME %s" % frame)
# parse the incoming request
# TODO: error handling
frame = json.loads(frame)
req = frame['req']
if req == 'startConsuming':
path = tuple(frame['path'])
options = frame['options']
# if it's already subscribed, don't leak a subscription
if path in self.qrefs:
return
def callback(key, message):
content = json.dumps(dict(path=path, key=key, message=message))
self.transport.write(content)
d = self.master.data.startConsuming(callback, options, path)
@d.addCallback
def register(qref):
if path in self.qrefs:
qref.stopConsuming()
self.qrefs[path] = qref
d.addErrback("while starting consumption")
def thd(
): # everything in deferToThread is not counted with trial --coverage :-(
url = self.tokenUri
data = {
'redirect_uri': self.loginUri,
'code': code,
'grant_type': self.grantType
}
auth = None
if self.getTokenUseAuthHeaders:
auth = (self.clientId, self.clientSecret)
else:
data.update({
'client_id': self.clientId,
'client_secret': self.clientSecret
})
data.update(self.tokenUriAdditionalParams)
response = requests.post(url,
data=data,
auth=auth,
verify=self.sslVerify)
response.raise_for_status()
if isinstance(response.content, basestring):
try:
content = json.loads(response.content)
except ValueError:
content = parse_qs(response.content)
for k, v in iteritems(content):
content[k] = v[0]
else:
content = response.content
session = self.createSessionFromToken(content)
return self.getUserInfoFromOAuthClient(session)
def assertStateByClass(self, name, class_name, **kwargs):
objectid = self.objects[(name, class_name)]
state = self.states[objectid]
for k, v in kwargs.iteritems():
self.t.assertIn(k, state)
self.t.assertEqual(json.loads(state[k]), v,
"state is %r" % (state,))
def _stepdictFromRow(self, row):
def mkdt(epoch):
if epoch:
return epoch2datetime(epoch)
return dict(
id=row.id,
number=row.number,
name=row.name,
buildid=row.buildid,
started_at=mkdt(row.started_at),
complete_at=mkdt(row.complete_at),
state_strings=json.loads(row.state_strings_json),
results=row.results,
urls=json.loads(row.urls_json),
)
def insertTestData(self, rows):
for row in rows:
if isinstance(row, Change):
ch = self.ChangeInstance()
ch.__dict__.update(dict(
# TODO: this renaming sucks.
number=row.changeid, who=row.author, files=[],
comments=row.comments, isdir=row.is_dir, links=[],
revision=row.revision, when=row.when_timestamp,
branch=row.branch, category=row.category,
revlink=row.revlink, properties=properties.Properties(),
repository=row.repository, project=row.project,
uid=None))
self.changes[row.changeid] = ch
elif isinstance(row, ChangeFile):
ch = self.changes[row.changeid]
ch.files.append(row.filename)
elif isinstance(row, ChangeLink):
ch = self.changes[row.changeid]
ch.links.append(row.link)
elif isinstance(row, ChangeProperty):
ch = self.changes[row.changeid]
n, vs = row.property_name, row.property_value
v, s = json.loads(vs)
ch.properties.setProperty(n, v, s)
elif isinstance(row, ChangeUser):
ch = self.changes[row.changeid]
ch.uid = row.uid
def lineReceived(self, line):
try:
event = json.loads(line.decode('utf-8'))
except ValueError:
log.msg("bad json line: %s" % (line,))
return defer.succeed(None)
if not(type(event) == type({}) and "type" in event):
log.msg("no type in event %s" % (line,))
return defer.succeed(None)
func = getattr(self, "eventReceived_"+event["type"].replace("-","_"), None)
if func == None:
log.msg("unsupported event %s" % (event["type"],))
return defer.succeed(None)
# flatten the event dictionary, for easy access with WithProperties
def flatten(properties, base, event):
for k, v in event.items():
if type(v) == dict:
flatten(properties, base + "." + k, v)
else: # already there
properties[base + "." + k] = v
properties = {}
flatten(properties, "event", event)
return func(properties,event)
def thd():
url = self.tokenUri
data = {'redirect_uri': self.loginUri, 'code': code,
'grant_type': self.grantType}
auth = None
if self.getTokenUseAuthHeaders:
auth = (self.clientId, self.clientSecret)
else:
data.update(
{'client_id': self.clientId, 'client_secret': self.clientSecret})
data.update(self.tokenUriAdditionalParams)
response = requests.post(
url, data=data, auth=auth, verify=self.sslVerify)
response.raise_for_status()
if isinstance(response.content, basestring):
try:
content = json.loads(response.content)
except ValueError:
content = parse_qs(response.content)
for k, v in iteritems(content):
content[k] = v[0]
else:
content = response.content
session = self.createSessionFromToken(content)
return self.getUserInfoFromOAuthClient(session)
def decodeJsonRPC2(self, request):
# Verify the content-type. Browsers are easily convinced to send
# POST data to arbitrary URLs via 'form' elements, but they won't
# use the application/json content-type.
if ContentTypeParser(request.getHeader("content-type")).gettype() != JSON_ENCODED:
raise BadJsonRpc2("Invalid content-type (use application/json)", JSONRPC_CODES["invalid_request"])
try:
data = json.loads(request.content.read())
except Exception as e:
raise BadJsonRpc2("JSON parse error: %s" % (str(e),), JSONRPC_CODES["parse_error"])
if isinstance(data, list):
raise BadJsonRpc2("JSONRPC batch requests are not supported", JSONRPC_CODES["invalid_request"])
if not isinstance(data, dict):
raise BadJsonRpc2("JSONRPC root object must be an object", JSONRPC_CODES["invalid_request"])
def check(name, types, typename):
if name not in data:
raise BadJsonRpc2("missing key '%s'" % (name,), JSONRPC_CODES["invalid_request"])
if not isinstance(data[name], types):
raise BadJsonRpc2("'%s' must be %s" % (name, typename), JSONRPC_CODES["invalid_request"])
check("jsonrpc", (str, unicode), "a string")
check("method", (str, unicode), "a string")
check("id", (str, unicode, int, type(None)), "a string, number, or null")
check("params", (dict,), "an object")
if data["jsonrpc"] != "2.0":
raise BadJsonRpc2("only JSONRPC 2.0 is supported", JSONRPC_CODES["invalid_request"])
return data["method"], data["id"], data["params"]
def assertRestCollection(self, typeName, items,
total=None, contentType=None, orderSignificant=False):
self.failIf(isinstance(self.request.written, unicode))
got = {}
got['content'] = json.loads(self.request.written)
got['contentType'] = self.request.headers['content-type']
got['responseCode'] = self.request.responseCode
meta = {}
if total is not None:
meta['total'] = total
exp = {}
exp['content'] = {typeName: items, 'meta': meta}
exp['contentType'] = [contentType or 'text/plain; charset=utf-8']
exp['responseCode'] = 200
# if order is not significant, sort so the comparison works
if not orderSignificant:
if 'content' in got and typeName in got['content']:
got['content'][typeName].sort()
exp['content'][typeName].sort()
if 'meta' in got['content'] and 'links' in got['content']['meta']:
got['content']['meta']['links'].sort(
key=lambda l: (l['rel'], l['href']))
self.assertEqual(got, exp)
def lineReceived(self, line):
try:
event = json.loads(line.decode('utf-8'))
except ValueError:
log.msg("bad json line: %s" % (line,))
return defer.succeed(None)
if not(type(event) == type({}) and "type" in event):
log.msg("no type in event %s" % (line,))
return defer.succeed(None)
func = getattr(self, "eventReceived_"+event["type"].replace("-","_"), None)
if func == None:
log.msg("unsupported event %s" % (event["type"],))
return defer.succeed(None)
# flatten the event dictionary, for easy access with WithProperties
def flatten(event, base, d):
for k, v in d.items():
if type(v) == dict:
flatten(event, base + "." + k, v)
else: # already there
event[base + "." + k] = v
properties = {}
flatten(properties, "event", event)
return func(properties,event)
def assertRestCollection(self, typeName, items, total=None, contentType=None, orderSignificant=False):
self.failIf(isinstance(self.request.written, unicode))
got = {}
got["content"] = json.loads(self.request.written)
got["contentType"] = self.request.headers["content-type"]
got["responseCode"] = self.request.responseCode
meta = {}
if total is not None:
meta["total"] = total
exp = {}
exp["content"] = {typeName: items, "meta": meta}
exp["contentType"] = [contentType or "text/plain; charset=utf-8"]
exp["responseCode"] = 200
# if order is not significant, sort so the comparison works
if not orderSignificant:
if "content" in got and typeName in got["content"]:
got["content"][typeName].sort()
exp["content"][typeName].sort()
if "meta" in got["content"] and "links" in got["content"]["meta"]:
got["content"]["meta"]["links"].sort(key=lambda l: (l["rel"], l["href"]))
self.assertEqual(got, exp)
def testDefaultDialectWithChange(self):
self.request.uri = "/change_hook/"
ret = self.changeHook.render_GET(self.request)
# Change is an array of dicts holding changes. There will normally only be one
# changes, thus only one dictionary
changeArray = json.loads(ret)
change = changeArray[0]
self.assertEquals(change["category"], "mycat")
files = change["files"]
self.assertEquals(len(files), 2)
self.assertEquals(files[0]["name"], "file1")
self.assertEquals(files[1]["name"], "file2")
self.assertEquals(change["repository"], "myrepo")
self.assertEquals(change["when"], 1234)
self.assertEquals(change["who"], "Santa Claus")
self.assertEquals(change["rev"], '99')
self.assertEquals(change["number"], None)
self.assertEquals(change["comments"], "a comment")
self.assertEquals(change["project"], "a project")
self.assertNotEquals(change["at"], "sometime")
self.assertEquals(change["branch"], "a branch")
self.assertEquals(change["revlink"], "a revlink")
properties = change["properties"]
self.assertEquals(len(properties), 2)
self.assertEquals(properties[0][0], "prop1")
self.assertEquals(properties[0][1], "val1")
self.assertEquals(properties[0][2], "Change")
self.assertEquals(properties[1][0], "prop2")
self.assertEquals(properties[1][1], "val2")
self.assertEquals(properties[1][2], "Change")
self.assertEquals(change["revision"], '99')
def assertStateByClass(self, name, class_name, **kwargs):
objectid = self.objects[(name, class_name)]
state = self.states[objectid]
for k, v in kwargs.iteritems():
self.t.assertIn(k, state)
self.t.assertEqual(json.loads(state[k]), v,
"state is %r" % (state, ))
def thd(conn):
bsp_tbl = self.db.model.buildset_properties
q = sa.select([bsp_tbl.c.property_name, bsp_tbl.c.property_value],
whereclause=(bsp_tbl.c.buildsetid == buildsetid))
return dict([(row.property_name,
tuple(json.loads(row.property_value)))
for row in conn.execute(q)])
def onMessage(self, frame, isBinary):
log.msg("FRAME %s" % frame)
# parse the incoming request
frame = json.loads(frame)
_id = frame.get("_id")
if _id is None:
return self.sendJsonMessage(error="no '_id' in websocket frame",
code=400,
_id=None)
cmd = frame.pop("cmd", None)
if cmd is None:
return self.sendJsonMessage(error="no 'cmd' in websocket frame",
code=400,
_id=None)
cmdmeth = "cmd_" + cmd
meth = getattr(self, cmdmeth, None)
if meth is None:
return self.sendJsonMessage(error="no such command '%s'" % (cmd, ),
code=404,
_id=_id)
try:
return meth(**frame)
except TypeError as e:
return self.sendJsonMessage(error="Invalid method argument '%s'" %
(str(e), ),
code=400,
_id=_id)
except Exception as e:
log.err("while calling command %s" % (cmd, ))
return self.sendJsonMessage(error="Internal Error '%s'" %
(str(e), ),
code=500,
_id=_id)
def assertRestCollection(self, typeName, items,
total=None, contentType=None, orderSignificant=False):
self.failIf(isinstance(self.request.written, unicode))
got = {}
got['content'] = json.loads(self.request.written)
got['contentType'] = self.request.headers['content-type']
got['responseCode'] = self.request.responseCode
meta = {}
if total is not None:
meta['total'] = total
exp = {}
exp['content'] = {typeName: items, 'meta': meta}
exp['contentType'] = [contentType or 'text/plain; charset=utf-8']
exp['responseCode'] = 200
# if order is not significant, sort so the comparison works
if not orderSignificant:
if 'content' in got and typeName in got['content']:
got['content'][typeName].sort()
exp['content'][typeName].sort()
if 'meta' in got['content'] and 'links' in got['content']['meta']:
got['content']['meta']['links'].sort(
key=lambda l: (l['rel'], l['href']))
self.assertEqual(got, exp)
def _chdict_from_change_row_thd(self, conn, ch_row):
# This method must be run in a db.pool thread, and returns a chdict
# given a row from the 'changes' table
change_links_tbl = self.db.model.change_links
change_files_tbl = self.db.model.change_files
change_properties_tbl = self.db.model.change_properties
def mkdt(epoch):
if epoch:
return epoch2datetime(epoch)
chdict = ChDict(
changeid=ch_row.changeid,
author=ch_row.author,
files=[], # see below
comments=ch_row.comments,
is_dir=ch_row.is_dir,
links=[], # see below
revision=ch_row.revision,
when_timestamp=mkdt(ch_row.when_timestamp),
branch=ch_row.branch,
category=ch_row.category,
revlink=ch_row.revlink,
properties={}, # see below
repository=ch_row.repository,
project=ch_row.project)
query = change_links_tbl.select(
whereclause=(change_links_tbl.c.changeid == ch_row.changeid))
rows = conn.execute(query)
for r in rows:
chdict['links'].append(r.link)
query = change_files_tbl.select(
whereclause=(change_files_tbl.c.changeid == ch_row.changeid))
rows = conn.execute(query)
for r in rows:
chdict['files'].append(r.filename)
# and properties must be given without a source, so strip that, but
# be flexible in case users have used a development version where the
# change properties were recorded incorrectly
def split_vs(vs):
try:
v, s = vs
if s != "Change":
v, s = vs, "Change"
except:
v, s = vs, "Change"
return v, s
query = change_properties_tbl.select(
whereclause=(change_properties_tbl.c.changeid == ch_row.changeid))
rows = conn.execute(query)
for r in rows:
v, s = split_vs(json.loads(r.property_value))
chdict['properties'][r.property_name] = (v, s)
return chdict
def assertState(self, objectid, missing_keys=[], **kwargs):
state = self.states[objectid]
for k in missing_keys:
self.t.assertFalse(k in state, "%s in %s" % (k, state))
for k, v in kwargs.iteritems():
self.t.assertIn(k, state)
self.t.assertEqual(json.loads(state[k]), v,
"state is %r" % (state,))
def assertState(self, objectid, missing_keys=[], **kwargs):
state = self.states[objectid]
for k in missing_keys:
self.t.assertFalse(k in state, "%s in %s" % (k, state))
for k, v in kwargs.iteritems():
self.t.assertIn(k, state)
self.t.assertEqual(json.loads(state[k]), v,
"state is %r" % (state, ))
def insertTestData(self, rows):
for row in rows:
if isinstance(row, Scheduler):
self.states[row.schedulerid] = json.loads(row.state)
elif isinstance(row, SchedulerChange):
cls = self.classifications.setdefault(row.schedulerid, {})
cls[row.changeid] = row.important
def validate(self, name, object):
if not isinstance(object, dict): # we want a dict, and NOT a subclass
yield "%s is not sourced properties (not a dict)" % (name, )
return
for k, v in object.iteritems():
if not isinstance(k, unicode):
yield "%s property name %r is not unicode" % (name, k)
if not isinstance(v, tuple) or len(v) != 2:
yield "%s property value for '%s' is not a 2-tuple" % (name, k)
return
propval, propsrc = v
if not isinstance(propsrc, unicode):
yield "%s[%s] source %r is not unicode" % (name, k, propsrc)
try:
json.loads(propval)
except ValueError:
yield "%s[%r] value is not JSON-able" % (name, k)
def getState(self, objectid, name, default=object):
try:
json_value = self.states[objectid][name]
except KeyError:
if default is not object:
return defer.succeed(default)
raise
return defer.succeed(json.loads(json_value))
def getState(self, objectid, name, default=object):
try:
json_value = self.states[objectid][name]
except KeyError:
if default is not object:
return defer.succeed(default)
raise
return defer.succeed(json.loads(json_value))
def check(_json):
res = json.loads(_json)
self.assertIn("jsonrpc", res)
self.assertEqual(res["jsonrpc"], "2.0")
if not requestJson:
# requestJson is used for invalid requests, so don't expect ID
self.assertIn("id", res)
self.assertEqual(res["id"], id)
def insertTestData(self, rows):
for row in rows:
if isinstance(row, Scheduler):
self.states[row.schedulerid] = json.loads(row.state)
elif isinstance(row, SchedulerChange):
cls = self.classifications.setdefault(row.schedulerid, {})
cls[row.changeid] = row.important
def thd(conn):
bsp_tbl = self.db.model.buildset_properties
q = sa.select(
[ bsp_tbl.c.property_name, bsp_tbl.c.property_value ],
whereclause=(bsp_tbl.c.buildsetid == buildsetid))
return dict([ (row.property_name,
tuple(json.loads(row.property_value)))
for row in conn.execute(q) ])
def validate(self, name, object):
if not isinstance(object, dict): # we want a dict, and NOT a subclass
yield "%s is not sourced properties (not a dict)" % (name,)
return
for k, v in iteritems(object):
if not isinstance(k, unicode):
yield "%s property name %r is not unicode" % (name, k)
if not isinstance(v, tuple) or len(v) != 2:
yield "%s property value for '%s' is not a 2-tuple" % (name, k)
return
propval, propsrc = v
if not isinstance(propsrc, unicode):
yield "%s[%s] source %r is not unicode" % (name, k, propsrc)
try:
json.loads(propval)
except ValueError:
yield "%s[%r] value is not JSON-able" % (name, k)
def decodeJsonRPC2(self, request):
# Content-Type is ignored, so that AJAX requests can be sent without
# incurring CORS preflight overheads. The JSONRPC spec does not
# suggest a Content-Type anyway.
try:
data = json.loads(request.content.read())
except Exception, e:
raise BadJsonRpc2("JSON parse error: %s" % (str(e), ),
JSONRPC_CODES["parse_error"])
def decodeJsonRPC2(self, request):
# Content-Type is ignored, so that AJAX requests can be sent without
# incurring CORS preflight overheads. The JSONRPC spec does not
# suggest a Content-Type anyway.
try:
data = json.loads(request.content.read())
except Exception, e:
raise BadJsonRpc2("JSON parse error: %s" % (str(e),),
JSONRPC_CODES["parse_error"])
def thd(conn):
bp_tbl = self.db.model.build_properties
q = sa.select([bp_tbl.c.name, bp_tbl.c.value, bp_tbl.c.source],
whereclause=(bp_tbl.c.buildid == bid))
props = []
for row in conn.execute(q):
prop = (json.loads(row.value), row.source)
props.append((row.name, prop))
return dict(props)
def lineReceived(self, line):
try:
event = json.loads(line.decode('utf-8'))
except ValueError:
log.msg("bad json line: %s" % (line,))
return defer.succeed(None)
if type(event) == type({}) and "type" in event and event["type"] in ["patchset-created", "ref-updated"]:
# flatten the event dictionary, for easy access with WithProperties
def flatten(event, base, d):
for k, v in d.items():
if type(v) == dict:
flatten(event, base + "." + k, v)
else: # already there
event[base + "." + k] = v
properties = {}
flatten(properties, "event", event)
if event["type"] == "patchset-created":
change = event["change"]
chdict = dict(
author="%s <%s>" % (change["owner"]["name"], change["owner"]["email"]),
project=change["project"],
branch=change["branch"],
revision=event["patchSet"]["revision"],
revlink=change["url"],
comments=change["subject"],
files=["unknown"],
category=event["type"],
properties=properties)
elif event["type"] == "ref-updated":
ref = event["refUpdate"]
author = "gerrit"
if "submitter" in event:
author="%s <%s>" % (event["submitter"]["name"], event["submitter"]["email"])
chdict = dict(
author=author,
project=ref["project"],
branch=ref["refName"],
revision=ref["newRev"],
comments="Gerrit: patchset(s) merged.",
files=["unknown"],
category=event["type"],
properties=properties)
else:
return defer.succeed(None) # this shouldn't happen anyway
d = self.master.addChange(**chdict)
# eat failures..
d.addErrback(log.err, 'error adding change from GerritChangeSource')
return d
else:
return defer.succeed(None)
def getJSON(data):
push = json.loads(data)
log.msg("Looking at the push json data for try comments")
for p in push:
pd = push[p]
changes = pd['changesets']
for change in reversed(changes):
match = re.search("try:", change['desc'])
if match:
return change['desc'].encode("utf8", "replace")
def __init__(self, headers, body, branch):
self._auth_code = headers['Google-Code-Project-Hosting-Hook-Hmac']
self._body = body # we need to save it if we want to authenticate it
self._branch = branch
payload = json.loads(body)
self.project = payload['project_name']
self.repository = payload['repository_path']
self.revisions = payload['revisions']
self.revision_count = payload['revision_count']
def __init__(self, headers, body, branch):
self._auth_code = headers['Google-Code-Project-Hosting-Hook-Hmac']
self._body = body # we need to save it if we want to authenticate it
self._branch = branch
payload = json.loads(body)
self.project = payload['project_name']
self.repository = payload['repository_path']
self.revisions = payload['revisions']
self.revision_count = payload['revision_count']
def assertRestError(self, responseCode, message):
got = {}
got['content'] = json.loads(self.request.written)
got['responseCode'] = self.request.responseCode
exp = {}
exp['content'] = {'error': message}
exp['responseCode'] = responseCode
self.assertEqual(got, exp)
def assertRestError(self, responseCode, message):
got = {}
got["content"] = json.loads(self.request.written)
got["responseCode"] = self.request.responseCode
exp = {}
exp["content"] = {"error": message}
exp["responseCode"] = responseCode
self.assertEqual(got, exp)
def thd(conn):
bp_tbl = self.db.model.build_properties
q = sa.select(
[bp_tbl.c.name, bp_tbl.c.value, bp_tbl.c.source],
whereclause=(bp_tbl.c.buildid == bid))
props = []
for row in conn.execute(q):
prop = (json.loads(row.value), row.source)
props.append((row.name, prop))
return dict(props)
def _txn_get_properties_from_db(self, t, tablename, idname, id):
# apparently you can't use argument placeholders for table names. Don't
# call this with a weird-looking tablename.
q = self.quoteq("SELECT property_name,property_value FROM %s WHERE %s=?" % (tablename, idname))
t.execute(q, (id,))
retval = Properties()
for key, valuepair in t.fetchall():
value, source = json.loads(valuepair)
retval.setProperty(str(key), value, source)
return retval
def assertRestError(self, responseCode, message):
got = {}
got['content'] = json.loads(self.request.written)
got['responseCode'] = self.request.responseCode
exp = {}
exp['content'] = {'error': message}
exp['responseCode'] = responseCode
self.assertEqual(got, exp)
def getJSON(data):
push = json.loads(data)
log.msg("Looking at the push json data for try comments")
for p in push:
pd = push[p]
changes = pd["changesets"]
for change in reversed(changes):
match = re.search("try:", change["desc"])
if match:
return change["desc"].encode("utf8", "replace")
def lineReceived(self, line):
try:
event = json.loads(line)
except ValueError:
log.msg("bad json line: %s" % (line, ))
return defer.succeed(None)
if type(event) == type({}) and "type" in event and event["type"] in [
"patchset-created", "ref-updated"
]:
# flatten the event dictionary, for easy access with WithProperties
def flatten(event, base, d):
for k, v in d.items():
if type(v) == dict:
flatten(event, base + "." + k, v)
else: # already there
event[base + "." + k] = v
properties = {}
flatten(properties, "event", event)
if event["type"] == "patchset-created":
change = event["change"]
chdict = dict(
who="%s <%s>" %
(change["owner"]["name"], change["owner"]["email"]),
project=change["project"],
branch=change["branch"],
revision=event["patchSet"]["revision"],
revlink=change["url"],
comments=change["subject"],
files=["unknown"],
category=event["type"],
properties=properties)
elif event["type"] == "ref-updated":
ref = event["refUpdate"]
chdict = dict(
who="%s <%s>" %
(event["submitter"]["name"], event["submitter"]["email"]),
project=ref["project"],
branch=ref["refName"],
revision=ref["newRev"],
comments="Gerrit: patchset(s) merged.",
files=["unknown"],
category=event["type"],
properties=properties)
else:
return defer.succeed(None) # this shouldn't happen anyway
d = self.master.addChange(**chdict)
# eat failures..
d.addErrback(log.err,
'error adding change from GerritChangeSource')
return d
def testGitWithChange(self):
self.request.uri = "/change_hook/github"
ret = self.changeHook.render_GET(self.request)
# Change is an array of dicts holding changes. There may be multiple entries for github
changeArray = json.loads(ret)
change = changeArray[0]
self.assertEquals(change["category"], None)
files = change["files"]
self.assertEquals(len(files), 1)
self.assertEquals(files[0]["name"], "filepath.rb")
self.assertEquals(change["repository"],
"http://github.com/defunkt/github")
self.assertEquals(change["when"], 1203116237)
self.assertEquals(change["who"], "Fred Flinstone <*****@*****.**>")
self.assertEquals(change["rev"],
'41a212ee83ca127e3c8cf465891ab7216a705f59')
self.assertEquals(change["number"], None)
self.assertEquals(change["comments"], "okay i give in")
self.assertEquals(change["project"], '')
self.assertNotEquals(change["at"], "sometime")
self.assertEquals(change["branch"], "master")
self.assertEquals(
change["revlink"],
"http://github.com/defunkt/github/commit/41a212ee83ca127e3c8cf465891ab7216a705f59"
)
properties = change["properties"]
self.assertEquals(len(properties), 0)
self.assertEquals(change["revision"],
'41a212ee83ca127e3c8cf465891ab7216a705f59')
# Second change
change = changeArray[1]
self.assertEquals(change["category"], None)
files = change["files"]
self.assertEquals(len(files), 2)
self.assertEquals(files[0]["name"], "modfile")
self.assertEquals(files[1]["name"], "removedFile")
self.assertEquals(change["repository"],
"http://github.com/defunkt/github")
self.assertEquals(change["when"], 1203114994)
self.assertEquals(change["who"], "Fred Flinstone <*****@*****.**>")
self.assertEquals(change["rev"],
'de8251ff97ee194a289832576287d6f8ad74e3d0')
self.assertEquals(change["number"], None)
self.assertEquals(change["comments"], "update pricing a tad")
self.assertEquals(change["project"], '')
self.assertNotEquals(change["at"], "sometime")
self.assertEquals(change["branch"], "master")
self.assertEquals(
change["revlink"],
"http://github.com/defunkt/github/commit/de8251ff97ee194a289832576287d6f8ad74e3d0"
)
properties = change["properties"]
self.assertEquals(len(properties), 0)
self.assertEquals(change["revision"],
'de8251ff97ee194a289832576287d6f8ad74e3d0')
def thd(conn):
schedulers_tbl = self.db.model.schedulers
q = sa.select([ schedulers_tbl.c.state ],
whereclause=(schedulers_tbl.c.schedulerid == schedulerid))
row = conn.execute(q).fetchone()
if not row:
return {} # really shouldn't happen - the row should exist
try:
return json.loads(row.state)
except:
log.msg("JSON error loading state for scheduler #%s" % (schedulerid,))
return {}
def _txn_get_properties_from_db(self, t, tablename, idname, id):
# apparently you can't use argument placeholders for table names. Don't
# call this with a weird-looking tablename.
q = self.quoteq(
"SELECT property_name,property_value FROM %s WHERE %s=?" %
(tablename, idname))
t.execute(q, (id, ))
retval = Properties()
for key, valuepair in t.fetchall():
value, source = json.loads(valuepair)
retval.setProperty(str(key), value, source)
return retval
def thd(conn):
bsp_tbl = self.db.model.buildset_properties
q = sa.select([bsp_tbl.c.property_name, bsp_tbl.c.property_value],
whereclause=(bsp_tbl.c.buildsetid == buildsetid))
l = []
for row in conn.execute(q):
try:
properties = json.loads(row.property_value)
l.append((row.property_name, tuple(properties)))
except ValueError:
pass
return dict(l)
def insertTestData(self, rows):
for row in rows:
if isinstance(row, Buildset):
bs = self.buildsets[row.id] = row.values.copy()
bs['properties'] = {}
for row in rows:
if isinstance(row, BuildsetProperty):
assert row.buildsetid in self.buildsets
n = row.property_name
v, src = tuple(json.loads(row.property_value))
self.buildsets[row.buildsetid]['properties'][n] = (v, src)
def thd(conn):
rows = self.getObjectStateData(conn, objects)
try:
object_state = {}
for row in rows:
object_state[row.name] = json.loads(row.value_json).keys()
return object_state
except:
raise TypeError("JSON error loading state value '%s'" %
(objects))
def decodeJsonRPC2(self, request):
# Verify the content-type. Browsers are easily convinced to send
# POST data to arbitrary URLs via 'form' elements, but they won't
# use the application/json content-type.
if request.getHeader('content-type') != 'application/json':
raise BadJsonRpc2('Invalid content-type (use application/json)',
JSONRPC_CODES["invalid_request"])
try:
data = json.loads(request.content.read())
except Exception, e:
raise BadJsonRpc2("JSON parse error: %s" % (str(e), ),
JSONRPC_CODES["parse_error"])
def thd(conn):
tbl = self.db.model.steps
wc = (tbl.c.id == stepid)
q = sa.select([tbl.c.urls_json], whereclause=wc)
res = conn.execute(q)
row = res.fetchone()
if _racehook is not None:
_racehook()
urls = json.loads(row.urls_json)
urls.append(dict(name=name, url=url))
q = tbl.update(whereclause=wc)
conn.execute(q, urls_json=json.dumps(urls))
def insertTestData(self, rows):
for row in rows:
if isinstance(row, Buildset):
bs = self.buildsets[row.id] = row.values.copy()
bs['properties'] = {}
if isinstance(row, SchedulerUpstreamBuildset):
self.buildset_subs.append((row.schedulerid, row.buildsetid))
for row in rows:
if isinstance(row, BuildsetProperty):
assert row.buildsetid in self.buildsets
n = row.property_name
v, src = tuple(json.loads(row.property_value))
self.buildsets[row.buildsetid]['properties'][n] = (v, src)
