Esempio n. 1
0
        def check_and_update_status(sha):
            # check that we haven't already completed this sha
            headers = {'User-agent': config.user_agent}
            conn, path, headers = tools.make_connection(self.db_url,
                                                        headers=headers)
            url = '%s/%s' % (self.db_url, sha)
            try:
                conn.request('GET', url, headers=headers)
            except socket.error as e:
                print 'socket.error:', e
                return
            response = conn.getresponse()
            payload = json.loads(response.read())

            if (('completed' in payload and payload['completed'])
                    or ('error' in payload)):
                return

            # loop over tests to see if we're done, and what the status is
            overall_success = True
            all_finished = True
            reason = 'Unknown'

            conn, path, headers = tools.make_connection(self.db_url,
                                                        headers=headers)
            query = 'startkey=["%s",1]&endkey=["%s",1,{}]' % (sha, sha)
            url = '%s/_design/pytunia/_view/tasks_by_record?%s' % (self.db_url,
                                                                   query)
            try:
                conn.request('GET', url, headers=headers)
            except socket.error as e:
                print 'socket.error:', e
                return
            response = conn.getresponse()
            payload = json.loads(response.read())['rows']

            for row in payload:
                doc = row['value']
                if 'error' in doc:
                    return

                if not 'completed' in doc:
                    all_finished = False
                    break

                if 'results' in doc and not doc['results']['success']:
                    overall_success = False
                    if reason == 'Unknown':
                        if 'reason' in doc['results']:
                            reason = doc['results']['reason']
                        else:
                            reason = 'Failed ' + doc['name']
                    else:
                        reason = 'Multiple tests failed'
                        break

            if all_finished:
                if overall_success:
                    status = 'success'
                    reason = 'All tests passed'
                else:
                    status = 'failure'

                print 'Updating status for commit %s: %s (%s)' % (sha, status,
                                                                  reason)

                url = config.results_base_url + sha
                code, body = PullRequestWatcher.set_commit_status(
                    config.user, self.repo, sha, status, reason, url,
                    self.oauth_token)

                if code >= 400:
                    print 'Error %i updating status on commit %s: %s' % (
                        code, sha, body)

                # update the document in the build test db to indicate completion
                conn, path, headers = tools.make_connection(self.db_url,
                                                            headers=headers)
                url = '%s/%s' % (self.db_url, sha)
                try:
                    conn.request('GET', url, headers=headers)
                except socket.error as e:
                    print 'socket.error:', e
                    return

                response = conn.getresponse()
                payload = json.loads(response.read())

                if 'error' in payload:
                    return

                payload['completed'] = True

                conn, path, headers = tools.make_connection(self.db_url,
                                                            headers=headers)
                url = '%s/%s' % (self.db_url, sha)
                try:
                    conn.request('PUT',
                                 url,
                                 json.dumps(payload),
                                 headers=headers)
                except socket.error as e:
                    print 'socket.error:', e
                    return
                response = conn.getresponse()
                if code != 201:
                    print 'Error %i updating DB for commit %s: %s' % (
                        code, sha, response.read())
Esempio n. 2
0
    def handle(self, doc):
        docs = []
        sha = doc['sha']

        # record (revision) document
        record = {
            '_id': sha,
            'type': 'record',
            'description': doc['message'],
            'created': time.time(),
            'author': doc['author'],
            'changeset_url': doc['pull_url']
        }

        docs.append(record)

        # cppcheck task document
        cppcheck = {
            '_id': uuid.uuid4().get_hex(),
            'type': 'task',
            'name': 'cppcheck',
            'created': time.time(),
            'platform': 'linux',
            'kwargs': {
                'base_repo_url': doc['base_repo_url'],
                'base_repo_ref': doc['base_repo_ref'],
                'git_url': doc['url'],
                'sha': sha,
            },
            'record_id': sha
        }

        docs.append(cppcheck)

        # fixme detector task document
        fixme = {
            '_id': uuid.uuid4().get_hex(),
            'type': 'task',
            'name': 'fixme',
            'created': time.time(),
            'platform': 'linux',
            'kwargs': {
                'base_repo_url': doc['base_repo_url'],
                'base_repo_ref': doc['base_repo_ref'],
                'git_url': doc['url'],
                'sha': sha,
            },
            'record_id': sha
        }

        docs.append(fixme)

        # size check task document
        sizecheck = {
            '_id': uuid.uuid4().get_hex(),
            'type': 'task',
            'name': 'size',
            'created': time.time(),
            'platform': 'linux',
            'kwargs': {
                'base_repo_url': doc['base_repo_url'],
                'base_repo_ref': doc['base_repo_ref'],
                'git_url': doc['url'],
                'sha': sha,
            },
            'record_id': sha
        }

        docs.append(sizecheck)

        # invalid character check task document
        charcheck = {
            '_id': uuid.uuid4().get_hex(),
            'type': 'task',
            'name': 'chartest',
            'created': time.time(),
            'platform': 'linux',
            'kwargs': {
                'base_repo_url': doc['base_repo_url'],
                'base_repo_ref': doc['base_repo_ref'],
                'git_url': doc['url'],
                'sha': sha,
            },
            'record_id': sha
        }

        docs.append(charcheck)

        # pylint task document
        pylint = {
            '_id': uuid.uuid4().get_hex(),
            'type': 'task',
            'name': 'pylint',
            'created': time.time(),
            'platform': 'linux',
            'kwargs': {
                'base_repo_url': doc['base_repo_url'],
                'base_repo_ref': doc['base_repo_ref'],
                'git_url': doc['url'],
                'sha': sha,
            },
            'record_id': sha
        }

        docs.append(pylint)

        # get task names with github api
        params = {}
        params['recursive'] = 1

        tree_base_url = '/repos/%s/git/trees/' % doc['full_name']

        conn = httplib.HTTPSConnection('api.github.com')
        try:
            req = conn.request('GET',
                               tree_base_url + sha + '?' +
                               urllib.urlencode(params),
                               headers=self.headers)
        except socket.error as e:
            print 'socket.error:', e
            return
        resp = conn.getresponse()
        data = resp.read()
        try:
            tree = json.loads(data)['tree']
        except ValueError:
            print 'ValueError:', data
            return

        tasknames = []
        for item in tree:
            p = self.test_path
            if item['type'] == 'tree' and item[
                    'path'][:len(p)] == p and item['path'] != p:
                tasknames.append(item['path'].split('/')[-1])

        # rattest task documents
        for taskname in tasknames:
            taskid = uuid.uuid4().get_hex()
            task = {
                '_id': taskid,
                'type': 'task',
                'name': 'rattest',
                'created': time.time(),
                'platform': 'linux',
                'kwargs': {
                    'base_repo_url': doc['base_repo_url'],
                    'base_repo_ref': doc['base_repo_ref'],
                    'git_url': doc['url'],
                    'sha': sha,
                    'testname': taskname
                },
                'record_id': sha
            }

            docs.append(task)

        # post documents to database
        headers = {'User-agent': config.user_agent}
        docs_json = json.dumps({'docs': docs})
        conn, path, headers = tools.make_connection(self.db_url,
                                                    headers=headers)
        try:
            conn.request('POST', '/%s/_bulk_docs' % path, docs_json, headers)
        except socket.error as e:
            print 'socket.error:', e
            return None, None
        response = conn.getresponse()

        user, repo = doc['full_name'].split('/')
        status = 'pending'
        url = self.results_base_url + sha
        description = 'Build %s started' % sha[:7]
        code, body = PullRequestWatcher.set_commit_status(
            config.user, repo, sha, status, description, url, self.oauth_token)
        if code != 201:
            print 'MergedPytuniaSubmitter: error %i setting commit status on %s:' % (
                code, sha)
            print body

        print 'MergedPytuniaSubmitter: pushed %i documents to %s for record %s, response: %s %s' % (
            len(docs), self.db_url, sha, response.status, response.reason)
Esempio n. 3
0
    def check_pulls(self):
        '''Generate pull request summary documents.

        :returns: Tuple of repo user, repo name, head sha, metadata document
        '''
        url = '/repos/%s/%s/pulls' % (self.user, self.repo)
        conn = httplib.HTTPSConnection('api.github.com')
        try:
            req = conn.request('GET', url, headers=self.headers)
        except socket.error as e:
            print 'Socket error:', e
            yield None, None, None, None
            return
        try:
            resp = conn.getresponse()
            data = resp.read()
        except Exception as e:
            print 'Exception: ' + str(e)
            yield None, None, None, None
            return

        try:
            payload = json.loads(data)
        except ValueError:
            print 'Malformed response:', data
            yield None, None, None, None
            return

        headers = {'User-agent': config.user_agent}
        for pr in payload:
            try:
                # format appropriately for MergedPytuniaSubmitter
                doc = {
                    'pull_url': pr['html_url'],
                    'base_repo_url': pr['base']['repo']['ssh_url'],
                    'base_repo_ref': pr['base']['ref'],
                    'url': pr['head']['repo']['ssh_url'],
                    'sha': pr['head']['sha'],
                    'full_name': pr['head']['repo']['full_name'],
                    'label': pr['head']['label'],
                    'message': pr['title'],
                    'author': pr['head']['user']['login']
                }

                user, repo = doc['full_name'].split('/')
                sha = doc['sha']

                conn, path, headers = tools.make_connection(self.db_url,
                                                            headers=headers)
                try:
                    conn.request('GET',
                                 '/pytunia-ondemand/' + sha,
                                 headers=headers)
                except socket.error as e:
                    print 'Socket error:', e
                    yield None, None, None, None
                    return
                status = conn.getresponse().status

                # only proceed on 404, i.e. no test of this sha exists
                if status == 401 or status == 403:
                    raise Exception('Authentication to builder failed')
                elif status < 400:
                    continue
                elif status != 404:
                    raise Exception('Unable to reach builder, error ' +
                                    status + ' occurred')

                yield user, repo, sha, doc
            except TypeError as e:
                print 'TypeError', e, pr
                pass
            except socket.error as e:
                print 'socket.error in MergedPytuniaSubmitter post', e
                pass
Esempio n. 4
0
        def check_and_update_status(sha):
            # check that we haven't already completed this sha
            headers = {'User-agent': config.user_agent}
            conn, path, headers = tools.make_connection(self.db_url, headers=headers)
            url = '%s/%s' % (self.db_url, sha)
            try:
                conn.request('GET', url, headers=headers)
            except socket.error as e:
                print 'socket.error:', e
                return
            response = conn.getresponse()
            payload = json.loads(response.read())

            if (('completed' in payload and payload['completed']) or
                ('error' in payload)):
                return

            # loop over tests to see if we're done, and what the status is
            overall_success = True
            all_finished = True
            reason = 'Unknown'

            conn, path, headers = tools.make_connection(self.db_url, headers=headers)
            query = 'startkey=["%s",1]&endkey=["%s",1,{}]' % (sha, sha)
            url = '%s/_design/pytunia/_view/tasks_by_record?%s' % (self.db_url, query)
            try:
                conn.request('GET', url, headers=headers)
            except socket.error as e:
                print 'socket.error:', e
                return
            response = conn.getresponse()
            payload = json.loads(response.read())['rows']

            for row in payload:
                doc = row['value']
                if 'error' in doc:
                    return

                if not 'completed' in doc:
                    all_finished = False
                    break

                if 'results' in doc and not doc['results']['success']:
                    overall_success = False
                    if reason == 'Unknown':
                        if 'reason' in doc['results']:
                            reason = doc['results']['reason']
                        else:
                            reason = 'Failed ' + doc['name']
                    else:
                        reason = 'Multiple tests failed'
                        break

            if all_finished:
                if overall_success:
                    status = 'success'
                    reason = 'All tests passed'
                else:
                    status = 'failure'

                print 'Updating status for commit %s: %s (%s)' % (sha, status, reason)

                url = config.results_base_url + sha
                code, body = PullRequestWatcher.set_commit_status(config.user, self.repo, sha, status, reason, url, self.params['access_token'])

                if code >= 400:
                    print 'Error %i updating status on commit %s: %s' % (code, sha, body)

                # update the document in the build test db to indicate completion
                conn, path, headers = tools.make_connection(self.db_url, headers=headers)
                url = '%s/%s' % (self.db_url, sha)
                try:
                    conn.request('GET', url, headers=headers)
                except socket.error as e:
                    print 'socket.error:', e
                    return
              
                response = conn.getresponse()
                payload = json.loads(response.read())

                if 'error' in payload:
                    return

                payload['completed'] = True

                conn, path, headers = tools.make_connection(self.db_url, headers=headers)
                url = '%s/%s' % (self.db_url, sha)
                try:
                    conn.request('PUT', url, json.dumps(payload), headers=headers)
                except socket.error as e:
                    print 'socket.error:', e
                    return
                response = conn.getresponse()
                if code != 201:
                    print 'Error %i updating DB for commit %s: %s' % (code, sha, response.read())
Esempio n. 5
0
    def handle(self, doc):
        docs = []
        sha = doc['sha']
        headers = {'User-agent': config.user_agent}

        # record (revision) document
        record = {
            '_id': sha,
            'type': 'record',
            'description': doc['message'],
            'created': time.time(),
            'author': doc['author'],
            'changeset_url': doc['pull_url']
        }

        docs.append(record)

        # cppcheck task document
        cppcheck = {
            '_id': uuid.uuid4().get_hex(),
            'type': 'task',
            'name': 'cppcheck',
            'created': time.time(),
            'platform': 'linux',
            'kwargs': {
                'base_repo_url': doc['base_repo_url'],
                'base_repo_ref': doc['base_repo_ref'],
                'git_url' : doc['url'],
                'sha': sha,
            },
            'record_id': sha
        }

        docs.append(cppcheck)

        # fixme detector task document
        fixme = {
            '_id': uuid.uuid4().get_hex(),
            'type': 'task',
            'name': 'fixme',
            'created': time.time(),
            'platform': 'linux',
            'kwargs': {
                'base_repo_url': doc['base_repo_url'],
                'base_repo_ref': doc['base_repo_ref'],
                'git_url' : doc['url'],
                'sha': sha,
            },
            'record_id': sha
        }

        docs.append(fixme)

        # size check task document
        sizecheck = {
            '_id': uuid.uuid4().get_hex(),
            'type': 'task',
            'name': 'size',
            'created': time.time(),
            'platform': 'linux',
            'kwargs': {
                'base_repo_url': doc['base_repo_url'],
                'base_repo_ref': doc['base_repo_ref'],
                'git_url' : doc['url'],
                'sha': sha,
            },
            'record_id': sha
        }

        docs.append(sizecheck)

        # invalid character check task document
        charcheck = {
            '_id': uuid.uuid4().get_hex(),
            'type': 'task',
            'name': 'chartest',
            'created': time.time(),
            'platform': 'linux',
            'kwargs': {
                'base_repo_url': doc['base_repo_url'],
                'base_repo_ref': doc['base_repo_ref'],
                'git_url' : doc['url'],
                'sha': sha,
            },
            'record_id': sha
        }

        docs.append(charcheck)

        # get task names with github api
        params = {'access_token': self.oauth_token} if self.oauth_token else {}
        params['recursive'] = 1

        tree_base_url = '/repos/%s/git/trees/' % doc['full_name']

        conn = httplib.HTTPSConnection('api.github.com')
        try:
            req = conn.request('GET', tree_base_url + sha + '?' + urllib.urlencode(params), headers=headers)
        except socket.error as e:
            print 'socket.error:', e
            return
        resp = conn.getresponse()
        data = resp.read()
        try:
            tree = json.loads(data)['tree']
        except ValueError:
            print 'ValueError:', data
            return

        tasknames = []
        for item in tree:
            p = self.test_path
            if item['type'] == 'tree' and item['path'][:len(p)] == p and item['path'] != p:
                tasknames.append(item['path'].split('/')[-1])

        # rattest task documents
        for taskname in tasknames:
            taskid = uuid.uuid4().get_hex()
            task = {
                '_id': taskid,
                'type': 'task',
                'name': 'rattest',
                'created': time.time(),
                'platform': 'linux',
                'kwargs': {
                    'base_repo_url': doc['base_repo_url'],
                    'base_repo_ref': doc['base_repo_ref'],
                    'git_url' : doc['url'],
                    'sha': sha,
                    'testname': taskname
                },
                'record_id': sha
            }

            docs.append(task)

        # post documents to database
        docs_json = json.dumps({'docs': docs})
        conn, path, headers = tools.make_connection(self.db_url, headers=headers)
        try:
            conn.request('POST', '/%s/_bulk_docs' % path, docs_json, headers)
        except socket.error as e:
            print 'socket.error:', e
            return None, None
        response = conn.getresponse()

        user, repo = doc['full_name'].split('/')
        status = 'pending'
        url = self.results_base_url + sha
        description = 'Build %s started' % sha[:7]
        code, body = PullRequestWatcher.set_commit_status(config.user, repo, sha, status, description, url, self.oauth_token)
        if code != 201:
            print 'MergedPytuniaSubmitter: error %i setting commit status on %s:' % (code, sha)
            print body

        print 'MergedPytuniaSubmitter: pushed %i documents to %s for record %s, response: %s %s' % (
            len(docs),
            self.db_url,
            sha,
            response.status,
            response.reason
        )
Esempio n. 6
0
    def check_pulls(self):
        '''Generate pull request summary documents.

        :returns: Tuple of repo user, repo name, head sha, metadata document
        '''
        url = '/repos/%s/%s/pulls' % (self.user, self.repo)
        conn = httplib.HTTPSConnection('api.github.com')
        headers = {'User-agent': config.user_agent}
        try:
            req = conn.request('GET', url + '?' + urllib.urlencode(self.params), headers=headers)
        except socket.error as e:
            print 'Socket error:', e
            yield None, None, None, None
            return
        try:
            resp = conn.getresponse()
            data = resp.read()
        except Exception as e:
            print 'Exception: ' + str(e)
            yield None, None, None, None
            return

        try:
            payload = json.loads(data)
        except ValueError:
            print 'Malformed response:', data
            yield None, None, None, None
            return

        for pr in payload:
            try:
                # format appropriately for MergedPytuniaSubmitter
                doc = {
                    'pull_url': pr['html_url'],
                    'base_repo_url': pr['base']['repo']['ssh_url'],
                    'base_repo_ref': pr['base']['ref'],
                    'url': pr['head']['repo']['ssh_url'],
                    'sha': pr['head']['sha'],
                    'full_name': pr['head']['repo']['full_name'],
                    'label': pr['head']['label'],
                    'message': pr['title'],
                    'author': pr['head']['user']['login']
                }

                user, repo = doc['full_name'].split('/')
                sha = doc['sha']

                conn, path, headers = tools.make_connection(self.db_url, headers=headers)
                try:
                    conn.request('GET', '/pytunia-ondemand/' + sha, headers=headers)
                except socket.error as e:
                    print 'Socket error:', e
                    yield None, None, None, None
                    return
                status = conn.getresponse().status

                # only proceed on 404, i.e. no test of this sha exists
                if status == 401 or status == 403:
                    raise Exception('Authentication to builder failed')
                elif status < 400:
                    continue
                elif status != 404:
                    raise Exception('Unable to reach builder, error ' + status + ' occurred')

                yield user, repo, sha, doc
            except TypeError as e:
                print 'TypeError', e, pr
                pass
            except socket.error as e:
                print 'socket.error in MergedPytuniaSubmitter post', e
                pass