Ejemplo n.º 1
0
 def assertReceivesChangeNewMessage(self, request):
     self.master.mq.callConsumer(("changes", "500", "new"), test_data_changes.Change.changeEvent)
     kw = self.readEvent(request)
     self.assertEqual(kw["event"], "event")
     msg = json.loads(kw["data"])
     self.assertEqual(msg["key"], [u'changes', u'500', u'new'])
     self.assertEqual(msg["message"], json.loads(json.dumps(test_data_changes.Change.changeEvent, default=self._toJson)))
Ejemplo n.º 2
0
    def _processChanges(self, page):
        result = json.loads(page, encoding=self.encoding)
        for pr in result["values"]:
            branch = pr["source"]["branch"]["name"]
            nr = int(pr["id"])
            # Note that this is a short hash. The full length hash can be accessed via the
            # commit api resource but we want to avoid requesting multiple pages as long as
            # we are not sure that the pull request is new or updated.
            revision = pr["source"]["commit"]["hash"]

            # check branch
            if not self.branch or branch in self.branch:
                current = yield self._getCurrentRev(nr)

                if not current or current != revision:
                    # parse pull request api page (required for the filter)
                    page = yield client.getPage(str(pr["links"]["self"]["href"]))
                    pr_json = json.loads(page, encoding=self.encoding)

                    # filter pull requests by user function
                    if not self.pullrequest_filter(pr_json):
                        log.msg("pull request does not match filter")
                        continue

                    # access additional information
                    author = pr["author"]["display_name"]
                    prlink = pr["links"]["html"]["href"]
                    # Get time updated time. Note that the timezone offset is ignored.
                    if self.useTimestamps:
                        updated = datetime.strptime(pr["updated_on"].split(".")[0], "%Y-%m-%dT%H:%M:%S")
                    else:
                        updated = epoch2datetime(reactor.seconds())
                    title = pr["title"]
                    # parse commit api page
                    page = yield client.getPage(str(pr["source"]["commit"]["links"]["self"]["href"]))
                    commit_json = json.loads(page, encoding=self.encoding)
                    # use the full-length hash from now on
                    revision = commit_json["hash"]
                    revlink = commit_json["links"]["html"]["href"]
                    # parse repo api page
                    page = yield client.getPage(str(pr["source"]["repository"]["links"]["self"]["href"]))
                    repo_json = json.loads(page, encoding=self.encoding)
                    repo = repo_json["links"]["html"]["href"]

                    # update database
                    yield self._setCurrentRev(nr, revision)
                    # emit the change
                    yield self.master.data.updates.addChange(
                        author=ascii2unicode(author),
                        revision=ascii2unicode(revision),
                        revlink=ascii2unicode(revlink),
                        comments=u"pull-request #%d: %s\n%s" % (nr, title, prlink),
                        when_timestamp=datetime2epoch(updated),
                        branch=self.branch,
                        category=self.category,
                        project=self.project,
                        repository=ascii2unicode(repo),
                        src=u"bitbucket",
                    )
Ejemplo n.º 3
0
def getChanges(request, options=None):
        """
        Consumes a naive build notification (the default for now)
        basically, set POST variables to match commit object parameters:
        revision, revlink, comments, branch, who, files, links
        
        files, links and properties will be de-json'd, the rest are interpreted as strings
        """
        
        def firstOrNothing( value ):
            """
            Small helper function to return the first value (if value is a list)
            or return the whole thing otherwise
            """
            if ( type(value) == type([])):
                return value[0]
            else:
                return value

        args = request.args

        # first, convert files, links and properties
        files = None
        if args.get('files'):
            files = json.loads( args.get('files')[0] )
        else:
            files = []
                
        links = None
        if args.get('links'):
            links = json.loads( args.get('links')[0] )
        else:
            links = []

        properties = None
        if args.get('properties'):
            properties = json.loads( args.get('properties')[0] )
        else:
            properties = {}
            
        revision = firstOrNothing(args.get('revision'))
        when     = firstOrNothing(args.get('when'))
        who = firstOrNothing(args.get('who'))
        comments = firstOrNothing(args.get('comments'))
        isdir = firstOrNothing(args.get('isdir',0))
        branch = firstOrNothing(args.get('branch'))
        category = firstOrNothing(args.get('category'))
        revlink = firstOrNothing(args.get('revlink'))
        repository = firstOrNothing(args.get('repository'))
        project = firstOrNothing(args.get('project'))
              
        chdict = dict(who=who, files=files, comments=comments,
                isdir=isdir, links=links, revision=revision, when=when,
                branch=branch, category=category, revlink=revlink,
                properties=properties, repository=repository,
                project=project)
        return ([ chdict ], None)
Ejemplo n.º 4
0
def _get_payload(request):
    content = request.content.read()
    content_type = request.getHeader(_HEADER_CT)

    if content_type == 'application/json':
        payload = json.loads(content)
    elif content_type == 'application/x-www-form-urlencoded':
        payload = json.loads(request.args['payload'][0])
    else:
        raise ValueError('Unknown content type: %r' % (content_type,))

    log.msg("Payload: %r" % payload, logLevel=logging.DEBUG)

    return payload
 def testDefaultDialectWithChange(self):
     self.request.uri = "/change_hook/"
     ret = self.changeHook.render_GET(self.request)
     # Change is an array of dicts holding changes. There will normally only be one
     # changes, thus only one dictionary
     changeArray = json.loads(ret)
     change = changeArray[0]
     self.assertEquals(change["category"], "mycat")
     files = change["files"]
     self.assertEquals(len(files), 2)
     self.assertEquals(files[0]["name"], "file1")
     self.assertEquals(files[1]["name"], "file2")
     self.assertEquals(change["repository"], "myrepo")
     self.assertEquals(change["when"], 1234)
     self.assertEquals(change["who"], "Santa Claus")
     self.assertEquals(change["rev"], "99")
     self.assertEquals(change["number"], None)
     self.assertEquals(change["comments"], "a comment")
     self.assertEquals(change["project"], "a project")
     self.assertNotEquals(change["at"], "sometime")
     self.assertEquals(change["branch"], "a branch")
     self.assertEquals(change["revlink"], "a revlink")
     properties = change["properties"]
     self.assertEquals(len(properties), 2)
     self.assertEquals(properties[0][0], "prop1")
     self.assertEquals(properties[0][1], "val1")
     self.assertEquals(properties[0][2], "Change")
     self.assertEquals(properties[1][0], "prop2")
     self.assertEquals(properties[1][1], "val2")
     self.assertEquals(properties[1][2], "Change")
     self.assertEquals(change["revision"], "99")
Ejemplo n.º 6
0
    def _chdict_from_change_row_thd(self, conn, ch_row):
        # This method must be run in a db.pool thread, and returns a chdict
        # given a row from the 'changes' table
        change_links_tbl = self.db.model.change_links
        change_files_tbl = self.db.model.change_files
        change_properties_tbl = self.db.model.change_properties

        def mkdt(epoch):
            if epoch:
                return epoch2datetime(epoch)

        chdict = ChDict(
            changeid=ch_row.changeid,
            author=ch_row.author,
            files=[],  # see below
            comments=ch_row.comments,
            is_dir=ch_row.is_dir,
            links=[],  # see below
            revision=ch_row.revision,
            when_timestamp=mkdt(ch_row.when_timestamp),
            branch=ch_row.branch,
            category=ch_row.category,
            revlink=ch_row.revlink,
            properties={},  # see below
            repository=ch_row.repository,
            project=ch_row.project,
        )

        query = change_links_tbl.select(whereclause=(change_links_tbl.c.changeid == ch_row.changeid))
        rows = conn.execute(query)
        for r in rows:
            chdict["links"].append(r.link)

        query = change_files_tbl.select(whereclause=(change_files_tbl.c.changeid == ch_row.changeid))
        rows = conn.execute(query)
        for r in rows:
            chdict["files"].append(r.filename)

        # and properties must be given without a source, so strip that, but
        # be flexible in case users have used a development version where the
        # change properties were recorded incorrectly
        def split_vs(vs):
            try:
                v, s = vs
                if s != "Change":
                    v, s = vs, "Change"
            except:
                v, s = vs, "Change"
            return v, s

        query = change_properties_tbl.select(whereclause=(change_properties_tbl.c.changeid == ch_row.changeid))
        rows = conn.execute(query)
        for r in rows:
            try:
                v, s = split_vs(json.loads(r.property_value))
                chdict["properties"][r.property_name] = (v, s)
            except ValueError:
                pass

        return chdict
Ejemplo n.º 7
0
    def dataReceived(self, frame):
        log.msg("FRAME %s" % frame)
        # parse the incoming request
        # TODO: error handling
        frame = json.loads(frame)
        req = frame['req']
        if req == 'startConsuming':
            path = tuple(frame['path'])
            options = frame['options']

            # if it's already subscribed, don't leak a subscription
            if path in self.qrefs:
                return

            def callback(key, message):
                content = json.dumps(dict(path=path, key=key, message=message))
                self.transport.write(content)

            d = self.master.data.startConsuming(callback, options, path)

            @d.addCallback
            def register(qref):
                if path in self.qrefs:
                    qref.stopConsuming()
                self.qrefs[path] = qref

            d.addErrback("while starting consumption")
Ejemplo n.º 8
0
    def insertTestData(self, rows):
        for row in rows:
            if isinstance(row, Change):
                ch = self.ChangeInstance()
                ch.__dict__.update(dict(
                    # TODO: this renaming sucks.
                    number=row.changeid, who=row.author, files=[],
                    comments=row.comments, isdir=row.is_dir, links=[],
                    revision=row.revision, when=row.when_timestamp,
                    branch=row.branch, category=row.category,
                    revlink=row.revlink, properties={},
                    repository=row.repository, project=row.project))
                self.changes[row.changeid] = ch

            elif isinstance(row, ChangeFile):
                ch = self.changes[row.changeid]
                ch.files.append(row.filename)

            elif isinstance(row, ChangeLink):
                ch = self.changes[row.changeid]
                ch.links.append(row.link)

            elif isinstance(row, ChangeProperty):
                ch = self.changes[row.changeid]
                n, v = row.property_name, row.property_value
                ch.properties[n] = json.loads(v)
Ejemplo n.º 9
0
 def _check(page):
     data = json.loads(page)
     self.assertEqual(len(data), 4)
     self.assertEqual(len(data["builders"]), 1)
     self.assertEqual(len(data["change_sources"]), 1)
     self.assertEqual(len(data["project"]), 2)
     self.assertEqual(len(data["slaves"]), 1)
Ejemplo n.º 10
0
    def dataReceived(self, frame):
        log.msg("FRAME %s" % frame)
        # parse the incoming request
        # TODO: error handling
        frame = json.loads(frame)
        req = frame['req']
        if req == 'startConsuming':
            path = tuple(frame['path'])
            options = frame['options']

            # if it's already subscribed, don't leak a subscription
            if path in self.qrefs:
                return

            def callback(key, message):
                content = json.dumps(dict(path=path, key=key, message=message))
                self.transport.write(content)
            d = self.master.data.startConsuming(callback, options, path)

            @d.addCallback
            def register(qref):
                if path in self.qrefs:
                    qref.stopConsuming()
                self.qrefs[path] = qref
            d.addErrback("while starting consumption")
Ejemplo n.º 11
0
        def thd(
        ):  # everything in deferToThread is not counted with trial  --coverage :-(
            url = self.tokenUri
            data = {
                'redirect_uri': self.loginUri,
                'code': code,
                'grant_type': self.grantType
            }
            auth = None
            if self.getTokenUseAuthHeaders:
                auth = (self.clientId, self.clientSecret)
            else:
                data.update({
                    'client_id': self.clientId,
                    'client_secret': self.clientSecret
                })
            data.update(self.tokenUriAdditionalParams)
            response = requests.post(url,
                                     data=data,
                                     auth=auth,
                                     verify=self.sslVerify)
            response.raise_for_status()
            if isinstance(response.content, basestring):
                try:
                    content = json.loads(response.content)
                except ValueError:
                    content = parse_qs(response.content)
                    for k, v in iteritems(content):
                        content[k] = v[0]
            else:
                content = response.content

            session = self.createSessionFromToken(content)
            return self.getUserInfoFromOAuthClient(session)
Ejemplo n.º 12
0
 def assertStateByClass(self, name, class_name, **kwargs):
     objectid = self.objects[(name, class_name)]
     state = self.states[objectid]
     for k, v in kwargs.iteritems():
         self.t.assertIn(k, state)
         self.t.assertEqual(json.loads(state[k]), v,
                            "state is %r" % (state,))
Ejemplo n.º 13
0
    def _stepdictFromRow(self, row):
        def mkdt(epoch):
            if epoch:
                return epoch2datetime(epoch)

        return dict(
            id=row.id,
            number=row.number,
            name=row.name,
            buildid=row.buildid,
            started_at=mkdt(row.started_at),
            complete_at=mkdt(row.complete_at),
            state_strings=json.loads(row.state_strings_json),
            results=row.results,
            urls=json.loads(row.urls_json),
        )
Ejemplo n.º 14
0
    def insertTestData(self, rows):
        for row in rows:
            if isinstance(row, Change):
                ch = self.ChangeInstance()
                ch.__dict__.update(dict(
                    # TODO: this renaming sucks.
                    number=row.changeid, who=row.author, files=[],
                    comments=row.comments, isdir=row.is_dir, links=[],
                    revision=row.revision, when=row.when_timestamp,
                    branch=row.branch, category=row.category,
                    revlink=row.revlink, properties=properties.Properties(),
                    repository=row.repository, project=row.project,
                    uid=None))
                self.changes[row.changeid] = ch

            elif isinstance(row, ChangeFile):
                ch = self.changes[row.changeid]
                ch.files.append(row.filename)

            elif isinstance(row, ChangeLink):
                ch = self.changes[row.changeid]
                ch.links.append(row.link)

            elif isinstance(row, ChangeProperty):
                ch = self.changes[row.changeid]
                n, vs = row.property_name, row.property_value
                v, s = json.loads(vs)
                ch.properties.setProperty(n, v, s)

            elif isinstance(row, ChangeUser):
                ch = self.changes[row.changeid]
                ch.uid = row.uid
Ejemplo n.º 15
0
    def lineReceived(self, line):
        try:
            event = json.loads(line.decode('utf-8'))
        except ValueError:
            log.msg("bad json line: %s" % (line,))
            return defer.succeed(None)

        if not(type(event) == type({}) and "type" in event):
            log.msg("no type in event %s" % (line,))
            return defer.succeed(None)
        func = getattr(self, "eventReceived_"+event["type"].replace("-","_"), None)
        if func == None:
            log.msg("unsupported event %s" % (event["type"],))
            return defer.succeed(None)

        # flatten the event dictionary, for easy access with WithProperties
        def flatten(properties, base, event):
            for k, v in event.items():
                if type(v) == dict:
                    flatten(properties, base + "." + k, v)
                else: # already there
                    properties[base + "." + k] = v

        properties = {}
        flatten(properties, "event", event)
        return func(properties,event)
Ejemplo n.º 16
0
        def thd():
            url = self.tokenUri
            data = {'redirect_uri': self.loginUri, 'code': code,
                    'grant_type': self.grantType}
            auth = None
            if self.getTokenUseAuthHeaders:
                auth = (self.clientId, self.clientSecret)
            else:
                data.update(
                    {'client_id': self.clientId, 'client_secret': self.clientSecret})
            data.update(self.tokenUriAdditionalParams)
            response = requests.post(
                url, data=data, auth=auth, verify=self.sslVerify)
            response.raise_for_status()
            if isinstance(response.content, basestring):
                try:
                    content = json.loads(response.content)
                except ValueError:
                    content = parse_qs(response.content)
                    for k, v in iteritems(content):
                        content[k] = v[0]
            else:
                content = response.content

            session = self.createSessionFromToken(content)
            return self.getUserInfoFromOAuthClient(session)
Ejemplo n.º 17
0
    def decodeJsonRPC2(self, request):
        # Verify the content-type.  Browsers are easily convinced to send
        # POST data to arbitrary URLs via 'form' elements, but they won't
        # use the application/json content-type.
        if ContentTypeParser(request.getHeader("content-type")).gettype() != JSON_ENCODED:
            raise BadJsonRpc2("Invalid content-type (use application/json)", JSONRPC_CODES["invalid_request"])

        try:
            data = json.loads(request.content.read())
        except Exception as e:
            raise BadJsonRpc2("JSON parse error: %s" % (str(e),), JSONRPC_CODES["parse_error"])

        if isinstance(data, list):
            raise BadJsonRpc2("JSONRPC batch requests are not supported", JSONRPC_CODES["invalid_request"])
        if not isinstance(data, dict):
            raise BadJsonRpc2("JSONRPC root object must be an object", JSONRPC_CODES["invalid_request"])

        def check(name, types, typename):
            if name not in data:
                raise BadJsonRpc2("missing key '%s'" % (name,), JSONRPC_CODES["invalid_request"])
            if not isinstance(data[name], types):
                raise BadJsonRpc2("'%s' must be %s" % (name, typename), JSONRPC_CODES["invalid_request"])

        check("jsonrpc", (str, unicode), "a string")
        check("method", (str, unicode), "a string")
        check("id", (str, unicode, int, type(None)), "a string, number, or null")
        check("params", (dict,), "an object")
        if data["jsonrpc"] != "2.0":
            raise BadJsonRpc2("only JSONRPC 2.0 is supported", JSONRPC_CODES["invalid_request"])
        return data["method"], data["id"], data["params"]
Ejemplo n.º 18
0
    def assertRestCollection(self, typeName, items,
                             total=None, contentType=None, orderSignificant=False):
        self.failIf(isinstance(self.request.written, unicode))
        got = {}
        got['content'] = json.loads(self.request.written)
        got['contentType'] = self.request.headers['content-type']
        got['responseCode'] = self.request.responseCode

        meta = {}
        if total is not None:
            meta['total'] = total

        exp = {}
        exp['content'] = {typeName: items, 'meta': meta}
        exp['contentType'] = [contentType or 'text/plain; charset=utf-8']
        exp['responseCode'] = 200

        # if order is not significant, sort so the comparison works
        if not orderSignificant:
            if 'content' in got and typeName in got['content']:
                got['content'][typeName].sort()
            exp['content'][typeName].sort()
        if 'meta' in got['content'] and 'links' in got['content']['meta']:
            got['content']['meta']['links'].sort(
                key=lambda l: (l['rel'], l['href']))

        self.assertEqual(got, exp)
Ejemplo n.º 19
0
    def lineReceived(self, line):
        try:
            event = json.loads(line.decode('utf-8'))
        except ValueError:
            log.msg("bad json line: %s" % (line,))
            return defer.succeed(None)

        if not(type(event) == type({}) and "type" in event):
            log.msg("no type in event %s" % (line,))
            return defer.succeed(None)
        func = getattr(self, "eventReceived_"+event["type"].replace("-","_"), None)
        if func == None:
            log.msg("unsupported event %s" % (event["type"],))
            return defer.succeed(None)

        # flatten the event dictionary, for easy access with WithProperties
        def flatten(event, base, d):
            for k, v in d.items():
                if type(v) == dict:
                    flatten(event, base + "." + k, v)
                else: # already there
                    event[base + "." + k] = v

        properties = {}
        flatten(properties, "event", event)
        return func(properties,event)
Ejemplo n.º 20
0
    def assertRestCollection(self, typeName, items, total=None, contentType=None, orderSignificant=False):
        self.failIf(isinstance(self.request.written, unicode))
        got = {}
        got["content"] = json.loads(self.request.written)
        got["contentType"] = self.request.headers["content-type"]
        got["responseCode"] = self.request.responseCode

        meta = {}
        if total is not None:
            meta["total"] = total

        exp = {}
        exp["content"] = {typeName: items, "meta": meta}
        exp["contentType"] = [contentType or "text/plain; charset=utf-8"]
        exp["responseCode"] = 200

        # if order is not significant, sort so the comparison works
        if not orderSignificant:
            if "content" in got and typeName in got["content"]:
                got["content"][typeName].sort()
            exp["content"][typeName].sort()
        if "meta" in got["content"] and "links" in got["content"]["meta"]:
            got["content"]["meta"]["links"].sort(key=lambda l: (l["rel"], l["href"]))

        self.assertEqual(got, exp)
Ejemplo n.º 21
0
 def testDefaultDialectWithChange(self):
     self.request.uri = "/change_hook/"
     ret = self.changeHook.render_GET(self.request)
     # Change is an array of dicts holding changes. There will normally only be one
     # changes, thus only one dictionary
     changeArray = json.loads(ret)
     change = changeArray[0]
     self.assertEquals(change["category"], "mycat")
     files = change["files"]
     self.assertEquals(len(files), 2)
     self.assertEquals(files[0]["name"], "file1")
     self.assertEquals(files[1]["name"], "file2")
     self.assertEquals(change["repository"], "myrepo")
     self.assertEquals(change["when"], 1234)
     self.assertEquals(change["who"], "Santa Claus")
     self.assertEquals(change["rev"], '99')
     self.assertEquals(change["number"], None)
     self.assertEquals(change["comments"], "a comment")
     self.assertEquals(change["project"], "a project")
     self.assertNotEquals(change["at"], "sometime")
     self.assertEquals(change["branch"], "a branch")
     self.assertEquals(change["revlink"], "a revlink")
     properties = change["properties"]
     self.assertEquals(len(properties), 2)
     self.assertEquals(properties[0][0], "prop1")
     self.assertEquals(properties[0][1], "val1")
     self.assertEquals(properties[0][2], "Change")
     self.assertEquals(properties[1][0], "prop2")
     self.assertEquals(properties[1][1], "val2")
     self.assertEquals(properties[1][2], "Change")
     self.assertEquals(change["revision"], '99')
Ejemplo n.º 22
0
 def assertStateByClass(self, name, class_name, **kwargs):
     objectid = self.objects[(name, class_name)]
     state = self.states[objectid]
     for k, v in kwargs.iteritems():
         self.t.assertIn(k, state)
         self.t.assertEqual(json.loads(state[k]), v,
                            "state is %r" % (state, ))
Ejemplo n.º 23
0
 def thd(conn):
     bsp_tbl = self.db.model.buildset_properties
     q = sa.select([bsp_tbl.c.property_name, bsp_tbl.c.property_value],
                   whereclause=(bsp_tbl.c.buildsetid == buildsetid))
     return dict([(row.property_name,
                   tuple(json.loads(row.property_value)))
                  for row in conn.execute(q)])
Ejemplo n.º 24
0
    def onMessage(self, frame, isBinary):
        log.msg("FRAME %s" % frame)
        # parse the incoming request

        frame = json.loads(frame)
        _id = frame.get("_id")
        if _id is None:
            return self.sendJsonMessage(error="no '_id' in websocket frame",
                                        code=400,
                                        _id=None)
        cmd = frame.pop("cmd", None)
        if cmd is None:
            return self.sendJsonMessage(error="no 'cmd' in websocket frame",
                                        code=400,
                                        _id=None)
        cmdmeth = "cmd_" + cmd
        meth = getattr(self, cmdmeth, None)
        if meth is None:
            return self.sendJsonMessage(error="no such command '%s'" % (cmd, ),
                                        code=404,
                                        _id=_id)
        try:
            return meth(**frame)
        except TypeError as e:
            return self.sendJsonMessage(error="Invalid method argument '%s'" %
                                        (str(e), ),
                                        code=400,
                                        _id=_id)
        except Exception as e:
            log.err("while calling command %s" % (cmd, ))
            return self.sendJsonMessage(error="Internal Error '%s'" %
                                        (str(e), ),
                                        code=500,
                                        _id=_id)
Ejemplo n.º 25
0
    def assertRestCollection(self, typeName, items,
                             total=None, contentType=None, orderSignificant=False):
        self.failIf(isinstance(self.request.written, unicode))
        got = {}
        got['content'] = json.loads(self.request.written)
        got['contentType'] = self.request.headers['content-type']
        got['responseCode'] = self.request.responseCode

        meta = {}
        if total is not None:
            meta['total'] = total

        exp = {}
        exp['content'] = {typeName: items, 'meta': meta}
        exp['contentType'] = [contentType or 'text/plain; charset=utf-8']
        exp['responseCode'] = 200

        # if order is not significant, sort so the comparison works
        if not orderSignificant:
            if 'content' in got and typeName in got['content']:
                got['content'][typeName].sort()
            exp['content'][typeName].sort()
        if 'meta' in got['content'] and 'links' in got['content']['meta']:
            got['content']['meta']['links'].sort(
                key=lambda l: (l['rel'], l['href']))

        self.assertEqual(got, exp)
Ejemplo n.º 26
0
    def _chdict_from_change_row_thd(self, conn, ch_row):
        # This method must be run in a db.pool thread, and returns a chdict
        # given a row from the 'changes' table
        change_links_tbl = self.db.model.change_links
        change_files_tbl = self.db.model.change_files
        change_properties_tbl = self.db.model.change_properties

        def mkdt(epoch):
            if epoch:
                return epoch2datetime(epoch)

        chdict = ChDict(
            changeid=ch_row.changeid,
            author=ch_row.author,
            files=[],  # see below
            comments=ch_row.comments,
            is_dir=ch_row.is_dir,
            links=[],  # see below
            revision=ch_row.revision,
            when_timestamp=mkdt(ch_row.when_timestamp),
            branch=ch_row.branch,
            category=ch_row.category,
            revlink=ch_row.revlink,
            properties={},  # see below
            repository=ch_row.repository,
            project=ch_row.project)

        query = change_links_tbl.select(
            whereclause=(change_links_tbl.c.changeid == ch_row.changeid))
        rows = conn.execute(query)
        for r in rows:
            chdict['links'].append(r.link)

        query = change_files_tbl.select(
            whereclause=(change_files_tbl.c.changeid == ch_row.changeid))
        rows = conn.execute(query)
        for r in rows:
            chdict['files'].append(r.filename)

        # and properties must be given without a source, so strip that, but
        # be flexible in case users have used a development version where the
        # change properties were recorded incorrectly
        def split_vs(vs):
            try:
                v, s = vs
                if s != "Change":
                    v, s = vs, "Change"
            except:
                v, s = vs, "Change"
            return v, s

        query = change_properties_tbl.select(
            whereclause=(change_properties_tbl.c.changeid == ch_row.changeid))
        rows = conn.execute(query)
        for r in rows:
            v, s = split_vs(json.loads(r.property_value))
            chdict['properties'][r.property_name] = (v, s)

        return chdict
Ejemplo n.º 27
0
 def assertState(self, objectid, missing_keys=[], **kwargs):
     state = self.states[objectid]
     for k in missing_keys:
         self.t.assertFalse(k in state, "%s in %s" % (k, state))
     for k, v in kwargs.iteritems():
         self.t.assertIn(k, state)
         self.t.assertEqual(json.loads(state[k]), v,
                            "state is %r" % (state,))
Ejemplo n.º 28
0
 def assertState(self, objectid, missing_keys=[], **kwargs):
     state = self.states[objectid]
     for k in missing_keys:
         self.t.assertFalse(k in state, "%s in %s" % (k, state))
     for k, v in kwargs.iteritems():
         self.t.assertIn(k, state)
         self.t.assertEqual(json.loads(state[k]), v,
                            "state is %r" % (state, ))
Ejemplo n.º 29
0
    def insertTestData(self, rows):
        for row in rows:
            if isinstance(row, Scheduler):
                self.states[row.schedulerid] = json.loads(row.state)

            elif isinstance(row, SchedulerChange):
                cls = self.classifications.setdefault(row.schedulerid, {})
                cls[row.changeid] = row.important
Ejemplo n.º 30
0
 def validate(self, name, object):
     if not isinstance(object, dict):  # we want a dict, and NOT a subclass
         yield "%s is not sourced properties (not a dict)" % (name, )
         return
     for k, v in object.iteritems():
         if not isinstance(k, unicode):
             yield "%s property name %r is not unicode" % (name, k)
         if not isinstance(v, tuple) or len(v) != 2:
             yield "%s property value for '%s' is not a 2-tuple" % (name, k)
             return
         propval, propsrc = v
         if not isinstance(propsrc, unicode):
             yield "%s[%s] source %r is not unicode" % (name, k, propsrc)
         try:
             json.loads(propval)
         except ValueError:
             yield "%s[%r] value is not JSON-able" % (name, k)
Ejemplo n.º 31
0
 def getState(self, objectid, name, default=object):
     try:
         json_value = self.states[objectid][name]
     except KeyError:
         if default is not object:
             return defer.succeed(default)
         raise
     return defer.succeed(json.loads(json_value))
Ejemplo n.º 32
0
 def getState(self, objectid, name, default=object):
     try:
         json_value = self.states[objectid][name]
     except KeyError:
         if default is not object:
             return defer.succeed(default)
         raise
     return defer.succeed(json.loads(json_value))
Ejemplo n.º 33
0
 def check(_json):
     res = json.loads(_json)
     self.assertIn("jsonrpc", res)
     self.assertEqual(res["jsonrpc"], "2.0")
     if not requestJson:
         # requestJson is used for invalid requests, so don't expect ID
         self.assertIn("id", res)
         self.assertEqual(res["id"], id)
Ejemplo n.º 34
0
    def insertTestData(self, rows):
        for row in rows:
            if isinstance(row, Scheduler):
                self.states[row.schedulerid] = json.loads(row.state)

            elif isinstance(row, SchedulerChange):
                cls = self.classifications.setdefault(row.schedulerid, {})
                cls[row.changeid] = row.important
Ejemplo n.º 35
0
 def thd(conn):
     bsp_tbl = self.db.model.buildset_properties
     q = sa.select(
         [ bsp_tbl.c.property_name, bsp_tbl.c.property_value ],
         whereclause=(bsp_tbl.c.buildsetid == buildsetid))
     return dict([ (row.property_name,
                    tuple(json.loads(row.property_value)))
                   for row in conn.execute(q) ])
Ejemplo n.º 36
0
 def validate(self, name, object):
     if not isinstance(object, dict):  # we want a dict, and NOT a subclass
         yield "%s is not sourced properties (not a dict)" % (name,)
         return
     for k, v in iteritems(object):
         if not isinstance(k, unicode):
             yield "%s property name %r is not unicode" % (name, k)
         if not isinstance(v, tuple) or len(v) != 2:
             yield "%s property value for '%s' is not a 2-tuple" % (name, k)
             return
         propval, propsrc = v
         if not isinstance(propsrc, unicode):
             yield "%s[%s] source %r is not unicode" % (name, k, propsrc)
         try:
             json.loads(propval)
         except ValueError:
             yield "%s[%r] value is not JSON-able" % (name, k)
Ejemplo n.º 37
0
 def decodeJsonRPC2(self, request):
     # Content-Type is ignored, so that AJAX requests can be sent without
     # incurring CORS preflight overheads.  The JSONRPC spec does not
     # suggest a Content-Type anyway.
     try:
         data = json.loads(request.content.read())
     except Exception, e:
         raise BadJsonRpc2("JSON parse error: %s" % (str(e), ),
                           JSONRPC_CODES["parse_error"])
Ejemplo n.º 38
0
 def decodeJsonRPC2(self, request):
     # Content-Type is ignored, so that AJAX requests can be sent without
     # incurring CORS preflight overheads.  The JSONRPC spec does not
     # suggest a Content-Type anyway.
     try:
         data = json.loads(request.content.read())
     except Exception, e:
         raise BadJsonRpc2("JSON parse error: %s" % (str(e),),
                           JSONRPC_CODES["parse_error"])
Ejemplo n.º 39
0
 def thd(conn):
     bp_tbl = self.db.model.build_properties
     q = sa.select([bp_tbl.c.name, bp_tbl.c.value, bp_tbl.c.source],
                   whereclause=(bp_tbl.c.buildid == bid))
     props = []
     for row in conn.execute(q):
         prop = (json.loads(row.value), row.source)
         props.append((row.name, prop))
     return dict(props)
Ejemplo n.º 40
0
    def lineReceived(self, line):
        try:
            event = json.loads(line.decode('utf-8'))
        except ValueError:
            log.msg("bad json line: %s" % (line,))
            return defer.succeed(None)

        if type(event) == type({}) and "type" in event and event["type"] in ["patchset-created", "ref-updated"]:
            # flatten the event dictionary, for easy access with WithProperties
            def flatten(event, base, d):
                for k, v in d.items():
                    if type(v) == dict:
                        flatten(event, base + "." + k, v)
                    else: # already there
                        event[base + "." + k] = v

            properties = {}
            flatten(properties, "event", event)

            if event["type"] == "patchset-created":
                change = event["change"]

                chdict = dict(
                        author="%s <%s>" % (change["owner"]["name"], change["owner"]["email"]),
                        project=change["project"],
                        branch=change["branch"],
                        revision=event["patchSet"]["revision"],
                        revlink=change["url"],
                        comments=change["subject"],
                        files=["unknown"],
                        category=event["type"],
                        properties=properties)
            elif event["type"] == "ref-updated":
                ref = event["refUpdate"]
                author = "gerrit"

                if "submitter" in event:
                    author="%s <%s>" % (event["submitter"]["name"], event["submitter"]["email"])

                chdict = dict(
                        author=author,
                        project=ref["project"],
                        branch=ref["refName"],
                        revision=ref["newRev"],
                        comments="Gerrit: patchset(s) merged.",
                        files=["unknown"],
                        category=event["type"],
                        properties=properties)
            else:
                return defer.succeed(None) # this shouldn't happen anyway

            d = self.master.addChange(**chdict)
            # eat failures..
            d.addErrback(log.err, 'error adding change from GerritChangeSource')
            return d
        else:
            return defer.succeed(None)
Ejemplo n.º 41
0
 def getJSON(data):
     push = json.loads(data)
     log.msg("Looking at the push json data for try comments")
     for p in push:
         pd = push[p]
         changes = pd['changesets']
         for change in reversed(changes):
             match = re.search("try:", change['desc'])
             if match:
                 return change['desc'].encode("utf8", "replace")
Ejemplo n.º 42
0
    def __init__(self, headers, body, branch):
        self._auth_code = headers['Google-Code-Project-Hosting-Hook-Hmac']
        self._body = body # we need to save it if we want to authenticate it
        self._branch = branch

        payload = json.loads(body)
        self.project = payload['project_name']
        self.repository = payload['repository_path']
        self.revisions = payload['revisions']
        self.revision_count = payload['revision_count']
Ejemplo n.º 43
0
    def __init__(self, headers, body, branch):
        self._auth_code = headers['Google-Code-Project-Hosting-Hook-Hmac']
        self._body = body  # we need to save it if we want to authenticate it
        self._branch = branch

        payload = json.loads(body)
        self.project = payload['project_name']
        self.repository = payload['repository_path']
        self.revisions = payload['revisions']
        self.revision_count = payload['revision_count']
Ejemplo n.º 44
0
    def assertRestError(self, responseCode, message):
        got = {}
        got['content'] = json.loads(self.request.written)
        got['responseCode'] = self.request.responseCode

        exp = {}
        exp['content'] = {'error': message}
        exp['responseCode'] = responseCode

        self.assertEqual(got, exp)
Ejemplo n.º 45
0
    def assertRestError(self, responseCode, message):
        got = {}
        got["content"] = json.loads(self.request.written)
        got["responseCode"] = self.request.responseCode

        exp = {}
        exp["content"] = {"error": message}
        exp["responseCode"] = responseCode

        self.assertEqual(got, exp)
Ejemplo n.º 46
0
 def thd(conn):
     bp_tbl = self.db.model.build_properties
     q = sa.select(
         [bp_tbl.c.name, bp_tbl.c.value, bp_tbl.c.source],
         whereclause=(bp_tbl.c.buildid == bid))
     props = []
     for row in conn.execute(q):
         prop = (json.loads(row.value), row.source)
         props.append((row.name, prop))
     return dict(props)
Ejemplo n.º 47
0
 def _txn_get_properties_from_db(self, t, tablename, idname, id):
     # apparently you can't use argument placeholders for table names. Don't
     # call this with a weird-looking tablename.
     q = self.quoteq("SELECT property_name,property_value FROM %s WHERE %s=?" % (tablename, idname))
     t.execute(q, (id,))
     retval = Properties()
     for key, valuepair in t.fetchall():
         value, source = json.loads(valuepair)
         retval.setProperty(str(key), value, source)
     return retval
Ejemplo n.º 48
0
    def assertRestError(self, responseCode, message):
        got = {}
        got['content'] = json.loads(self.request.written)
        got['responseCode'] = self.request.responseCode

        exp = {}
        exp['content'] = {'error': message}
        exp['responseCode'] = responseCode

        self.assertEqual(got, exp)
Ejemplo n.º 49
0
 def getJSON(data):
     push = json.loads(data)
     log.msg("Looking at the push json data for try comments")
     for p in push:
         pd = push[p]
         changes = pd["changesets"]
         for change in reversed(changes):
             match = re.search("try:", change["desc"])
             if match:
                 return change["desc"].encode("utf8", "replace")
Ejemplo n.º 50
0
    def lineReceived(self, line):
        try:
            event = json.loads(line)
        except ValueError:
            log.msg("bad json line: %s" % (line, ))
            return defer.succeed(None)

        if type(event) == type({}) and "type" in event and event["type"] in [
                "patchset-created", "ref-updated"
        ]:
            # flatten the event dictionary, for easy access with WithProperties
            def flatten(event, base, d):
                for k, v in d.items():
                    if type(v) == dict:
                        flatten(event, base + "." + k, v)
                    else:  # already there
                        event[base + "." + k] = v

            properties = {}
            flatten(properties, "event", event)

            if event["type"] == "patchset-created":
                change = event["change"]

                chdict = dict(
                    who="%s <%s>" %
                    (change["owner"]["name"], change["owner"]["email"]),
                    project=change["project"],
                    branch=change["branch"],
                    revision=event["patchSet"]["revision"],
                    revlink=change["url"],
                    comments=change["subject"],
                    files=["unknown"],
                    category=event["type"],
                    properties=properties)
            elif event["type"] == "ref-updated":
                ref = event["refUpdate"]
                chdict = dict(
                    who="%s <%s>" %
                    (event["submitter"]["name"], event["submitter"]["email"]),
                    project=ref["project"],
                    branch=ref["refName"],
                    revision=ref["newRev"],
                    comments="Gerrit: patchset(s) merged.",
                    files=["unknown"],
                    category=event["type"],
                    properties=properties)
            else:
                return defer.succeed(None)  # this shouldn't happen anyway

            d = self.master.addChange(**chdict)
            # eat failures..
            d.addErrback(log.err,
                         'error adding change from GerritChangeSource')
            return d
 def testGitWithChange(self):
     self.request.uri = "/change_hook/github"
     ret = self.changeHook.render_GET(self.request)
     # Change is an array of dicts holding changes. There may be multiple entries for github
     changeArray = json.loads(ret)
     change = changeArray[0]
     self.assertEquals(change["category"], None)
     files = change["files"]
     self.assertEquals(len(files), 1)
     self.assertEquals(files[0]["name"], "filepath.rb")
     self.assertEquals(change["repository"],
                       "http://github.com/defunkt/github")
     self.assertEquals(change["when"], 1203116237)
     self.assertEquals(change["who"], "Fred Flinstone <*****@*****.**>")
     self.assertEquals(change["rev"],
                       '41a212ee83ca127e3c8cf465891ab7216a705f59')
     self.assertEquals(change["number"], None)
     self.assertEquals(change["comments"], "okay i give in")
     self.assertEquals(change["project"], '')
     self.assertNotEquals(change["at"], "sometime")
     self.assertEquals(change["branch"], "master")
     self.assertEquals(
         change["revlink"],
         "http://github.com/defunkt/github/commit/41a212ee83ca127e3c8cf465891ab7216a705f59"
     )
     properties = change["properties"]
     self.assertEquals(len(properties), 0)
     self.assertEquals(change["revision"],
                       '41a212ee83ca127e3c8cf465891ab7216a705f59')
     # Second change
     change = changeArray[1]
     self.assertEquals(change["category"], None)
     files = change["files"]
     self.assertEquals(len(files), 2)
     self.assertEquals(files[0]["name"], "modfile")
     self.assertEquals(files[1]["name"], "removedFile")
     self.assertEquals(change["repository"],
                       "http://github.com/defunkt/github")
     self.assertEquals(change["when"], 1203114994)
     self.assertEquals(change["who"], "Fred Flinstone <*****@*****.**>")
     self.assertEquals(change["rev"],
                       'de8251ff97ee194a289832576287d6f8ad74e3d0')
     self.assertEquals(change["number"], None)
     self.assertEquals(change["comments"], "update pricing a tad")
     self.assertEquals(change["project"], '')
     self.assertNotEquals(change["at"], "sometime")
     self.assertEquals(change["branch"], "master")
     self.assertEquals(
         change["revlink"],
         "http://github.com/defunkt/github/commit/de8251ff97ee194a289832576287d6f8ad74e3d0"
     )
     properties = change["properties"]
     self.assertEquals(len(properties), 0)
     self.assertEquals(change["revision"],
                       'de8251ff97ee194a289832576287d6f8ad74e3d0')
Ejemplo n.º 52
0
 def thd(conn):
     schedulers_tbl = self.db.model.schedulers
     q = sa.select([ schedulers_tbl.c.state ],
             whereclause=(schedulers_tbl.c.schedulerid == schedulerid))
     row = conn.execute(q).fetchone()
     if not row:
         return {} # really shouldn't happen - the row should exist
     try:
         return json.loads(row.state)
     except:
         log.msg("JSON error loading state for scheduler #%s" % (schedulerid,))
         return {}
Ejemplo n.º 53
0
 def _txn_get_properties_from_db(self, t, tablename, idname, id):
     # apparently you can't use argument placeholders for table names. Don't
     # call this with a weird-looking tablename.
     q = self.quoteq(
         "SELECT property_name,property_value FROM %s WHERE %s=?" %
         (tablename, idname))
     t.execute(q, (id, ))
     retval = Properties()
     for key, valuepair in t.fetchall():
         value, source = json.loads(valuepair)
         retval.setProperty(str(key), value, source)
     return retval
Ejemplo n.º 54
0
 def thd(conn):
     bsp_tbl = self.db.model.buildset_properties
     q = sa.select([bsp_tbl.c.property_name, bsp_tbl.c.property_value],
                   whereclause=(bsp_tbl.c.buildsetid == buildsetid))
     l = []
     for row in conn.execute(q):
         try:
             properties = json.loads(row.property_value)
             l.append((row.property_name, tuple(properties)))
         except ValueError:
             pass
     return dict(l)
Ejemplo n.º 55
0
    def insertTestData(self, rows):
        for row in rows:
            if isinstance(row, Buildset):
                bs = self.buildsets[row.id] = row.values.copy()
                bs['properties'] = {}

        for row in rows:
            if isinstance(row, BuildsetProperty):
                assert row.buildsetid in self.buildsets
                n = row.property_name
                v, src = tuple(json.loads(row.property_value))
                self.buildsets[row.buildsetid]['properties'][n] = (v, src)
Ejemplo n.º 56
0
        def thd(conn):
            rows = self.getObjectStateData(conn, objects)

            try:
                object_state = {}
                for row in rows:
                    object_state[row.name] = json.loads(row.value_json).keys()

                return object_state
            except:
                raise TypeError("JSON error loading state value '%s'" %
                                (objects))
Ejemplo n.º 57
0
    def decodeJsonRPC2(self, request):
        # Verify the content-type.  Browsers are easily convinced to send
        # POST data to arbitrary URLs via 'form' elements, but they won't
        # use the application/json content-type.
        if request.getHeader('content-type') != 'application/json':
            raise BadJsonRpc2('Invalid content-type (use application/json)',
                              JSONRPC_CODES["invalid_request"])

        try:
            data = json.loads(request.content.read())
        except Exception, e:
            raise BadJsonRpc2("JSON parse error: %s" % (str(e), ),
                              JSONRPC_CODES["parse_error"])
Ejemplo n.º 58
0
        def thd(conn):

            tbl = self.db.model.steps
            wc = (tbl.c.id == stepid)
            q = sa.select([tbl.c.urls_json], whereclause=wc)
            res = conn.execute(q)
            row = res.fetchone()
            if _racehook is not None:
                _racehook()
            urls = json.loads(row.urls_json)
            urls.append(dict(name=name, url=url))

            q = tbl.update(whereclause=wc)
            conn.execute(q, urls_json=json.dumps(urls))
Ejemplo n.º 59
0
    def insertTestData(self, rows):
        for row in rows:
            if isinstance(row, Buildset):
                bs = self.buildsets[row.id] = row.values.copy()
                bs['properties'] = {}
            if isinstance(row, SchedulerUpstreamBuildset):
                self.buildset_subs.append((row.schedulerid, row.buildsetid))

        for row in rows:
            if isinstance(row, BuildsetProperty):
                assert row.buildsetid in self.buildsets
                n = row.property_name
                v, src = tuple(json.loads(row.property_value))
                self.buildsets[row.buildsetid]['properties'][n] = (v, src)