def check_and_update_status(sha): # check that we haven't already completed this sha headers = {'User-agent': config.user_agent} conn, path, headers = tools.make_connection(self.db_url, headers=headers) url = '%s/%s' % (self.db_url, sha) try: conn.request('GET', url, headers=headers) except socket.error as e: print 'socket.error:', e return response = conn.getresponse() payload = json.loads(response.read()) if (('completed' in payload and payload['completed']) or ('error' in payload)): return # loop over tests to see if we're done, and what the status is overall_success = True all_finished = True reason = 'Unknown' conn, path, headers = tools.make_connection(self.db_url, headers=headers) query = 'startkey=["%s",1]&endkey=["%s",1,{}]' % (sha, sha) url = '%s/_design/pytunia/_view/tasks_by_record?%s' % (self.db_url, query) try: conn.request('GET', url, headers=headers) except socket.error as e: print 'socket.error:', e return response = conn.getresponse() payload = json.loads(response.read())['rows'] for row in payload: doc = row['value'] if 'error' in doc: return if not 'completed' in doc: all_finished = False break if 'results' in doc and not doc['results']['success']: overall_success = False if reason == 'Unknown': if 'reason' in doc['results']: reason = doc['results']['reason'] else: reason = 'Failed ' + doc['name'] else: reason = 'Multiple tests failed' break if all_finished: if overall_success: status = 'success' reason = 'All tests passed' else: status = 'failure' print 'Updating status for commit %s: %s (%s)' % (sha, status, reason) url = config.results_base_url + sha code, body = PullRequestWatcher.set_commit_status( config.user, self.repo, sha, status, reason, url, self.oauth_token) if code >= 400: print 'Error %i updating status on commit %s: %s' % ( code, sha, body) # update the document in the build test db to indicate completion conn, path, headers = tools.make_connection(self.db_url, headers=headers) url = '%s/%s' % (self.db_url, sha) try: conn.request('GET', url, headers=headers) except socket.error as e: print 'socket.error:', e return response = conn.getresponse() payload = json.loads(response.read()) if 'error' in payload: return payload['completed'] = True conn, path, headers = tools.make_connection(self.db_url, headers=headers) url = '%s/%s' % (self.db_url, sha) try: conn.request('PUT', url, json.dumps(payload), headers=headers) except socket.error as e: print 'socket.error:', e return response = conn.getresponse() if code != 201: print 'Error %i updating DB for commit %s: %s' % ( code, sha, response.read())
def handle(self, doc): docs = [] sha = doc['sha'] # record (revision) document record = { '_id': sha, 'type': 'record', 'description': doc['message'], 'created': time.time(), 'author': doc['author'], 'changeset_url': doc['pull_url'] } docs.append(record) # cppcheck task document cppcheck = { '_id': uuid.uuid4().get_hex(), 'type': 'task', 'name': 'cppcheck', 'created': time.time(), 'platform': 'linux', 'kwargs': { 'base_repo_url': doc['base_repo_url'], 'base_repo_ref': doc['base_repo_ref'], 'git_url': doc['url'], 'sha': sha, }, 'record_id': sha } docs.append(cppcheck) # fixme detector task document fixme = { '_id': uuid.uuid4().get_hex(), 'type': 'task', 'name': 'fixme', 'created': time.time(), 'platform': 'linux', 'kwargs': { 'base_repo_url': doc['base_repo_url'], 'base_repo_ref': doc['base_repo_ref'], 'git_url': doc['url'], 'sha': sha, }, 'record_id': sha } docs.append(fixme) # size check task document sizecheck = { '_id': uuid.uuid4().get_hex(), 'type': 'task', 'name': 'size', 'created': time.time(), 'platform': 'linux', 'kwargs': { 'base_repo_url': doc['base_repo_url'], 'base_repo_ref': doc['base_repo_ref'], 'git_url': doc['url'], 'sha': sha, }, 'record_id': sha } docs.append(sizecheck) # invalid character check task document charcheck = { '_id': uuid.uuid4().get_hex(), 'type': 'task', 'name': 'chartest', 'created': time.time(), 'platform': 'linux', 'kwargs': { 'base_repo_url': doc['base_repo_url'], 'base_repo_ref': doc['base_repo_ref'], 'git_url': doc['url'], 'sha': sha, }, 'record_id': sha } docs.append(charcheck) # pylint task document pylint = { '_id': uuid.uuid4().get_hex(), 'type': 'task', 'name': 'pylint', 'created': time.time(), 'platform': 'linux', 'kwargs': { 'base_repo_url': doc['base_repo_url'], 'base_repo_ref': doc['base_repo_ref'], 'git_url': doc['url'], 'sha': sha, }, 'record_id': sha } docs.append(pylint) # get task names with github api params = {} params['recursive'] = 1 tree_base_url = '/repos/%s/git/trees/' % doc['full_name'] conn = httplib.HTTPSConnection('api.github.com') try: req = conn.request('GET', tree_base_url + sha + '?' + urllib.urlencode(params), headers=self.headers) except socket.error as e: print 'socket.error:', e return resp = conn.getresponse() data = resp.read() try: tree = json.loads(data)['tree'] except ValueError: print 'ValueError:', data return tasknames = [] for item in tree: p = self.test_path if item['type'] == 'tree' and item[ 'path'][:len(p)] == p and item['path'] != p: tasknames.append(item['path'].split('/')[-1]) # rattest task documents for taskname in tasknames: taskid = uuid.uuid4().get_hex() task = { '_id': taskid, 'type': 'task', 'name': 'rattest', 'created': time.time(), 'platform': 'linux', 'kwargs': { 'base_repo_url': doc['base_repo_url'], 'base_repo_ref': doc['base_repo_ref'], 'git_url': doc['url'], 'sha': sha, 'testname': taskname }, 'record_id': sha } docs.append(task) # post documents to database headers = {'User-agent': config.user_agent} docs_json = json.dumps({'docs': docs}) conn, path, headers = tools.make_connection(self.db_url, headers=headers) try: conn.request('POST', '/%s/_bulk_docs' % path, docs_json, headers) except socket.error as e: print 'socket.error:', e return None, None response = conn.getresponse() user, repo = doc['full_name'].split('/') status = 'pending' url = self.results_base_url + sha description = 'Build %s started' % sha[:7] code, body = PullRequestWatcher.set_commit_status( config.user, repo, sha, status, description, url, self.oauth_token) if code != 201: print 'MergedPytuniaSubmitter: error %i setting commit status on %s:' % ( code, sha) print body print 'MergedPytuniaSubmitter: pushed %i documents to %s for record %s, response: %s %s' % ( len(docs), self.db_url, sha, response.status, response.reason)
def check_pulls(self): '''Generate pull request summary documents. :returns: Tuple of repo user, repo name, head sha, metadata document ''' url = '/repos/%s/%s/pulls' % (self.user, self.repo) conn = httplib.HTTPSConnection('api.github.com') try: req = conn.request('GET', url, headers=self.headers) except socket.error as e: print 'Socket error:', e yield None, None, None, None return try: resp = conn.getresponse() data = resp.read() except Exception as e: print 'Exception: ' + str(e) yield None, None, None, None return try: payload = json.loads(data) except ValueError: print 'Malformed response:', data yield None, None, None, None return headers = {'User-agent': config.user_agent} for pr in payload: try: # format appropriately for MergedPytuniaSubmitter doc = { 'pull_url': pr['html_url'], 'base_repo_url': pr['base']['repo']['ssh_url'], 'base_repo_ref': pr['base']['ref'], 'url': pr['head']['repo']['ssh_url'], 'sha': pr['head']['sha'], 'full_name': pr['head']['repo']['full_name'], 'label': pr['head']['label'], 'message': pr['title'], 'author': pr['head']['user']['login'] } user, repo = doc['full_name'].split('/') sha = doc['sha'] conn, path, headers = tools.make_connection(self.db_url, headers=headers) try: conn.request('GET', '/pytunia-ondemand/' + sha, headers=headers) except socket.error as e: print 'Socket error:', e yield None, None, None, None return status = conn.getresponse().status # only proceed on 404, i.e. no test of this sha exists if status == 401 or status == 403: raise Exception('Authentication to builder failed') elif status < 400: continue elif status != 404: raise Exception('Unable to reach builder, error ' + status + ' occurred') yield user, repo, sha, doc except TypeError as e: print 'TypeError', e, pr pass except socket.error as e: print 'socket.error in MergedPytuniaSubmitter post', e pass
def check_and_update_status(sha): # check that we haven't already completed this sha headers = {'User-agent': config.user_agent} conn, path, headers = tools.make_connection(self.db_url, headers=headers) url = '%s/%s' % (self.db_url, sha) try: conn.request('GET', url, headers=headers) except socket.error as e: print 'socket.error:', e return response = conn.getresponse() payload = json.loads(response.read()) if (('completed' in payload and payload['completed']) or ('error' in payload)): return # loop over tests to see if we're done, and what the status is overall_success = True all_finished = True reason = 'Unknown' conn, path, headers = tools.make_connection(self.db_url, headers=headers) query = 'startkey=["%s",1]&endkey=["%s",1,{}]' % (sha, sha) url = '%s/_design/pytunia/_view/tasks_by_record?%s' % (self.db_url, query) try: conn.request('GET', url, headers=headers) except socket.error as e: print 'socket.error:', e return response = conn.getresponse() payload = json.loads(response.read())['rows'] for row in payload: doc = row['value'] if 'error' in doc: return if not 'completed' in doc: all_finished = False break if 'results' in doc and not doc['results']['success']: overall_success = False if reason == 'Unknown': if 'reason' in doc['results']: reason = doc['results']['reason'] else: reason = 'Failed ' + doc['name'] else: reason = 'Multiple tests failed' break if all_finished: if overall_success: status = 'success' reason = 'All tests passed' else: status = 'failure' print 'Updating status for commit %s: %s (%s)' % (sha, status, reason) url = config.results_base_url + sha code, body = PullRequestWatcher.set_commit_status(config.user, self.repo, sha, status, reason, url, self.params['access_token']) if code >= 400: print 'Error %i updating status on commit %s: %s' % (code, sha, body) # update the document in the build test db to indicate completion conn, path, headers = tools.make_connection(self.db_url, headers=headers) url = '%s/%s' % (self.db_url, sha) try: conn.request('GET', url, headers=headers) except socket.error as e: print 'socket.error:', e return response = conn.getresponse() payload = json.loads(response.read()) if 'error' in payload: return payload['completed'] = True conn, path, headers = tools.make_connection(self.db_url, headers=headers) url = '%s/%s' % (self.db_url, sha) try: conn.request('PUT', url, json.dumps(payload), headers=headers) except socket.error as e: print 'socket.error:', e return response = conn.getresponse() if code != 201: print 'Error %i updating DB for commit %s: %s' % (code, sha, response.read())
def handle(self, doc): docs = [] sha = doc['sha'] headers = {'User-agent': config.user_agent} # record (revision) document record = { '_id': sha, 'type': 'record', 'description': doc['message'], 'created': time.time(), 'author': doc['author'], 'changeset_url': doc['pull_url'] } docs.append(record) # cppcheck task document cppcheck = { '_id': uuid.uuid4().get_hex(), 'type': 'task', 'name': 'cppcheck', 'created': time.time(), 'platform': 'linux', 'kwargs': { 'base_repo_url': doc['base_repo_url'], 'base_repo_ref': doc['base_repo_ref'], 'git_url' : doc['url'], 'sha': sha, }, 'record_id': sha } docs.append(cppcheck) # fixme detector task document fixme = { '_id': uuid.uuid4().get_hex(), 'type': 'task', 'name': 'fixme', 'created': time.time(), 'platform': 'linux', 'kwargs': { 'base_repo_url': doc['base_repo_url'], 'base_repo_ref': doc['base_repo_ref'], 'git_url' : doc['url'], 'sha': sha, }, 'record_id': sha } docs.append(fixme) # size check task document sizecheck = { '_id': uuid.uuid4().get_hex(), 'type': 'task', 'name': 'size', 'created': time.time(), 'platform': 'linux', 'kwargs': { 'base_repo_url': doc['base_repo_url'], 'base_repo_ref': doc['base_repo_ref'], 'git_url' : doc['url'], 'sha': sha, }, 'record_id': sha } docs.append(sizecheck) # invalid character check task document charcheck = { '_id': uuid.uuid4().get_hex(), 'type': 'task', 'name': 'chartest', 'created': time.time(), 'platform': 'linux', 'kwargs': { 'base_repo_url': doc['base_repo_url'], 'base_repo_ref': doc['base_repo_ref'], 'git_url' : doc['url'], 'sha': sha, }, 'record_id': sha } docs.append(charcheck) # get task names with github api params = {'access_token': self.oauth_token} if self.oauth_token else {} params['recursive'] = 1 tree_base_url = '/repos/%s/git/trees/' % doc['full_name'] conn = httplib.HTTPSConnection('api.github.com') try: req = conn.request('GET', tree_base_url + sha + '?' + urllib.urlencode(params), headers=headers) except socket.error as e: print 'socket.error:', e return resp = conn.getresponse() data = resp.read() try: tree = json.loads(data)['tree'] except ValueError: print 'ValueError:', data return tasknames = [] for item in tree: p = self.test_path if item['type'] == 'tree' and item['path'][:len(p)] == p and item['path'] != p: tasknames.append(item['path'].split('/')[-1]) # rattest task documents for taskname in tasknames: taskid = uuid.uuid4().get_hex() task = { '_id': taskid, 'type': 'task', 'name': 'rattest', 'created': time.time(), 'platform': 'linux', 'kwargs': { 'base_repo_url': doc['base_repo_url'], 'base_repo_ref': doc['base_repo_ref'], 'git_url' : doc['url'], 'sha': sha, 'testname': taskname }, 'record_id': sha } docs.append(task) # post documents to database docs_json = json.dumps({'docs': docs}) conn, path, headers = tools.make_connection(self.db_url, headers=headers) try: conn.request('POST', '/%s/_bulk_docs' % path, docs_json, headers) except socket.error as e: print 'socket.error:', e return None, None response = conn.getresponse() user, repo = doc['full_name'].split('/') status = 'pending' url = self.results_base_url + sha description = 'Build %s started' % sha[:7] code, body = PullRequestWatcher.set_commit_status(config.user, repo, sha, status, description, url, self.oauth_token) if code != 201: print 'MergedPytuniaSubmitter: error %i setting commit status on %s:' % (code, sha) print body print 'MergedPytuniaSubmitter: pushed %i documents to %s for record %s, response: %s %s' % ( len(docs), self.db_url, sha, response.status, response.reason )
def check_pulls(self): '''Generate pull request summary documents. :returns: Tuple of repo user, repo name, head sha, metadata document ''' url = '/repos/%s/%s/pulls' % (self.user, self.repo) conn = httplib.HTTPSConnection('api.github.com') headers = {'User-agent': config.user_agent} try: req = conn.request('GET', url + '?' + urllib.urlencode(self.params), headers=headers) except socket.error as e: print 'Socket error:', e yield None, None, None, None return try: resp = conn.getresponse() data = resp.read() except Exception as e: print 'Exception: ' + str(e) yield None, None, None, None return try: payload = json.loads(data) except ValueError: print 'Malformed response:', data yield None, None, None, None return for pr in payload: try: # format appropriately for MergedPytuniaSubmitter doc = { 'pull_url': pr['html_url'], 'base_repo_url': pr['base']['repo']['ssh_url'], 'base_repo_ref': pr['base']['ref'], 'url': pr['head']['repo']['ssh_url'], 'sha': pr['head']['sha'], 'full_name': pr['head']['repo']['full_name'], 'label': pr['head']['label'], 'message': pr['title'], 'author': pr['head']['user']['login'] } user, repo = doc['full_name'].split('/') sha = doc['sha'] conn, path, headers = tools.make_connection(self.db_url, headers=headers) try: conn.request('GET', '/pytunia-ondemand/' + sha, headers=headers) except socket.error as e: print 'Socket error:', e yield None, None, None, None return status = conn.getresponse().status # only proceed on 404, i.e. no test of this sha exists if status == 401 or status == 403: raise Exception('Authentication to builder failed') elif status < 400: continue elif status != 404: raise Exception('Unable to reach builder, error ' + status + ' occurred') yield user, repo, sha, doc except TypeError as e: print 'TypeError', e, pr pass except socket.error as e: print 'socket.error in MergedPytuniaSubmitter post', e pass