def _get_client(self, username=None, password=None, anonymous=False):
from rbtools.api.client import RBClient
from rbtools.api.transport.sync import SyncTransport
class NoCacheTransport(SyncTransport):
"""API transport with disabled caching."""
def enable_cache(self):
pass
# TODO consider moving this to __init__.
if not self.mr:
raise Exception('Could not find MozReview cluster instance')
if username is None or password is None:
username = os.environ.get('BUGZILLA_USERNAME')
password = os.environ.get('BUGZILLA_PASSWORD')
# RBClient is persisting login cookies from call to call
# in $HOME/.rbtools-cookies. We want to be able to easily switch
# between users, so we clear that cookie between calls to the
# server and reauthenticate every time.
try:
os.remove(os.path.join(os.environ.get('HOME'), '.rbtools-cookies'))
except Exception:
pass
if anonymous:
return RBClient(self.mr.reviewboard_url,
transport_cls=NoCacheTransport)
return RBClient(self.mr.reviewboard_url, username=username,
password=password, transport_cls=NoCacheTransport)
def get_root():
settings = get_config()
review = settings['reviewboard']
client = RBClient(review['server'])
client.login(review['user'], review['password'])
return client.get_root()
def ReviewBoardClient(url, username=None, password=None, apikey=None):
"""Obtain a RBClient instance."""
if username and apikey:
rbc = RBClient(url, save_cookies=False, allow_caching=False)
login_resource = rbc.get_path(
'extensions/mozreview.extension.MozReviewExtension/'
'bugzilla-api-key-logins/')
login_resource.create(username=username, api_key=apikey)
else:
rbc = RBClient(url,
username=username,
password=password,
save_cookies=False,
allow_caching=False)
return rbc
def download_reviews(self, dev_group):
self.dev_group = dev_group
root = RBClient(self.url, api_token=self.token).get_root()
reviews = root.get_review_requests(to_groups=dev_group, status='pending', counts_only=True)
reviews = root.get_review_requests(to_groups=dev_group, status='pending', max_results=reviews.count)
self.reviews = [self.RawReviewInfo(r) for r in reviews]
def get_rb_client(self):
if not self.rb_client:
options = {}
with open(".reviewboardrc") as reviewboardrc:
for line in reviewboardrc:
if line.startswith("#"):
continue
if len(line.strip()) == 0:
continue
k, v = line.strip().split("=")
k = k.strip()
v = eval(v.strip())
options[k] = v
rbclient = RBClient(options['REVIEWBOARD_URL'])
self.repository = options['REPOSITORY']
self.branch = options.get('BRANCH') or options.get('TRACKING_BRANCH')
self.target_groups = None
if options.has_key('TARGET_GROUPS'):
self.target_groups = options['TARGET_GROUPS']
if rbclient.get_root().get_session()['authenticated']:
return rbclient.get_root()
username = self.opt.reviewboard_username or raw_input("Enter review board Username: "******"Enter password: ")
rbclient.login(username, password)
self.rb_client = rbclient.get_root()
return self.rb_client
def setUp(self):
super(SVNRepositoryInfoTests, self).setUp()
self.spy_on(urlopen, call_fake=self._urlopen)
self.api_client = RBClient('http://localhost:8080/')
self.root_resource = self.api_client.get_root()
def get_api(server_url, **kwargs):
"""Returns an RBClient instance and the associated root resource.
Hooks should use this method to gain access to the API, instead of
instantiating their own client.
Args:
server_url (unicode):
The server URL to retrieve.
**kwargs (dict):
Additional keyword arguments to pass to the
:py:class:`~rbtools.api.client.RBClient` constructor. See
:py:meth:`SyncTransport.__init__()
<rbtools.api.transport.sync.SyncTransport.__init__>` for arguments
that are accepted.
Returns:
tuple:
This returns a 2-tuple of the :py:class:`~rbtools.api.client.RBClient`
and :py:class:`<root resource> rbtools.api.resource.Resource`.
"""
api_client = RBClient(server_url, **kwargs)
try:
api_root = api_client.get_root()
except ServerInterfaceError as e:
raise HookError('Could not reach the Review Board server at %s: %s' %
(server_url, e))
except APIError as e:
raise HookError('Unexpected API Error: %s' % e)
return api_client, api_root
def setUp(self):
super(RepositoryMatchTests, self).setUp()
@self.spy_for(urlopen)
def _urlopen(url, **kwargs):
url = url.get_full_url()
try:
payload = self.payloads[url]
except KeyError:
print('Test requested unexpected URL "%s"' % url)
return MockResponse(
404, {},
json.dumps({
'rsp': {
'stat': 'fail',
'err': {
'code': 100,
'msg': 'Object does not exist',
},
},
}))
return MockResponse(200, {
'Content-Type': payload['mimetype'],
}, json.dumps(payload['rsp']))
self.api_client = RBClient('http://localhost:8080/')
self.root_resource = self.api_client.get_root()
def __init__(self, configuration, *args):
'''
Helper to build an RBTools api root
used by MozReview below
'''
url, api_key, username = self.requires(configuration, 'url', 'api_key',
'username')
# Authenticate client
client = RBClient(url, save_cookies=False, allow_caching=False)
login_resource = client.get_path(
'extensions/mozreview.extension.MozReviewExtension/'
'bugzilla-api-key-logins/')
login_resource.create(username=username, api_key=api_key)
self.api = client.get_root()
# Report issues from specific analyzers
self.analyzers = list(
filter(
lambda a: a in (CLANG_TIDY, CLANG_FORMAT, MOZLINT),
configuration.get('analyzers', [
CLANG_TIDY,
]),
))
assert len(self.analyzers) > 0, \
'No valid analyzers for mozreview'
self.publish_success = configuration.get('publish_success', False)
assert isinstance(self.publish_success, bool)
logger.info('Mozreview report enabled',
url=url,
username=username,
analyzers=self.analyzers)
def rb_client(self):
options = {}
if os.path.exists(".reviewboardrc"):
with open(".reviewboardrc") as reviewboardrc:
for line in reviewboardrc:
if line.startswith("#"):
continue
if len(line.strip()) == 0:
continue
k, v = line.strip().split("=")
k = k.strip()
v = eval(v.strip())
options[k] = v
rbclient = RBClient(
options.get('REVIEWBOARD_URL') or 'https://reviews.apache.org/',
RetryingSyncTransport)
if not rbclient.get_root().get_session()['authenticated']:
username = self.opt.reviewboard_username[0] if self.opt.reviewboard_username and \
self.opt.reviewboard_username[0] else raw_input(
"Enter review board Username: "******"Enter password for %s: " % username)
rbclient.login(username, password)
root = rbclient.get_root()
root.repository = options.get('REPOSITORY') or None
root.branch = options.get('BRANCH') or options.get('TRACKING_BRANCH')
root.target_groups = None
if options.has_key('TARGET_GROUPS'):
root.target_groups = options['TARGET_GROUPS']
return root
def setUp(self):
super(SVNRepositoryInfoTests, self).setUp()
def _urlopen(url, **kwargs):
url = url.get_full_url()
try:
payload = self.payloads[url]
except KeyError:
return MockResponse(
404, {},
json.dumps({
'rsp': {
'stat': 'fail',
'err': {
'code': 100,
'msg': 'Object does not exist',
},
},
}))
return MockResponse(200, {
'Content-Type': payload['mimetype'],
}, json.dumps(payload['rsp']))
self.spy_on(urlopen, call_fake=_urlopen)
self.api_client = RBClient('http://localhost:8080/')
self.root_resource = self.api_client.get_root()
def ship_it(self, rrid, username, password):
"""Create and publish a ship-it review"""
# TODO: this code is lifted from the reviewboard mach commands. If we
# need to make more reviewboard api calls in these tests, we
# should make a function in the base class to get a RBClient.
class NoCacheTransport(SyncTransport):
"""API transport with disabled caching."""
def enable_cache(self):
pass
# RBClient is persisting login cookies from call to call
# in $HOME/.rbtools-cookies. We want to be able to easily switch
# between users, so we clear that cookie between calls to the
# server and reauthenticate every time.
try:
os.remove(os.path.join(os.environ.get('HOME'), '.rbtools-cookies'))
except Exception:
pass
client = RBClient(self.rburl,
username=username,
password=password,
transport_cls=NoCacheTransport)
root = client.get_root()
reviews = root.get_reviews(review_request_id=rrid)
reviews.create(public=True, ship_it=True)
def _make_api_client(self, server_url):
"""Return an RBClient object for the server.
The RBClient will be instantiated with the proper arguments
for talking to the provided Review Board server url.
"""
return RBClient(server_url,
username=self.options.username,
password=self.options.password,
auth_callback=self.credentials_prompt)
def main():
'''
Posts a review to the review with specified id indicating if the build
passed or failed
'''
parser = argparse.ArgumentParser()
parser.add_argument("--cfg", required=True, help="Configuration file")
parser.add_argument("--reviewid",
required=True,
help="Review id to post to")
parser.add_argument("--buildurl",
required=True,
help="The jenkins build url")
parser.add_argument("--buildstate",
required=True,
choices=["SUCCESS", "UNSTABLE", "FAILURE"],
help="Indicates if build succeeded (1) or failed (0)")
args = parser.parse_args()
reviewid = args.reviewid
buildurl = args.buildurl
buildstate = args.buildstate
config = ConfigParser.ConfigParser()
try:
config.read(args.cfg)
client = RBClient(config.get('jrbb', 'reviewboard_server'),
username=config.get('jrbb', 'reviewboard_user'),
api_token=config.get('jrbb', 'reviewboard_apitoken'))
except ConfigParser.NoSectionError:
print "Configuration file " + args.cfg + " not found or missing items"
exit(1)
root = client.get_root()
# Get the revision number of the latest diff from the review
# pylint: disable=no-member
reviewreq = root.get_review_request(review_request_id=reviewid)
reviews = reviewreq.get_reviews()
if buildstate == 'SUCCESS':
msg = 'Successfully built changes. See ' + buildurl
else:
msg = 'Opps! I could not build these changes. See ' + buildurl
reviews.create(body_bottom=msg, public=True)
print "Posted to review " + reviewid + " build state=" + buildstate + \
". Build url=" + buildurl
def __init__(self, url):
self.client = RBClient(url)
self.root = self.client.get_root()
try:
self._version = float(
self.root.rsp['product']['version'].split()[0])
except:
self._version = 0.0
self._templates = self.root.rsp['uri_templates']
self._files = {}
self._file_data = {}
self._simplefile_data = {}
def build_api_root(url, username, api_key):
'''
Helper to build an RBTools api root
used by BatchReview
'''
logger.info('Authenticate on Mozreview', url=url, username=username)
client = RBClient(url, save_cookies=False, allow_caching=False)
login_resource = client.get_path(
'extensions/mozreview.extension.MozReviewExtension/'
'bugzilla-api-key-logins/')
login_resource.create(username=username, api_key=api_key)
return client.get_root()
def run(self):
while True:
id = q.get()
client = RBClient(reviewboard_url, api_token=api_token)
root = client.get_root()
review = root.get_review_request(review_request_id=id)
comments = review.get_reviews(max_results=200)
for i, c in enumerate(comments):
if i == len(comments) - 1:
break
if c.links.user.title == jenkins_uname and c.ship_it:
c.update(ship_it=False)
def get_api(server_url, username, password):
"""Returns an RBClient instance and the associated root resource.
Hooks should use this method to gain access to the API, instead of
instantianting their own client.
"""
api_client = RBClient(server_url, username=username, password=password)
try:
api_root = api_client.get_root()
except ServerInterfaceError, e:
raise HookError('Could not reach the Review Board server at %s: %s' %
(server_url, e))
def main():
'''
Fetches the latest patch diff from the review given the review id passed
as a parameter and writes it an output file.
'''
parser = argparse.ArgumentParser()
parser.add_argument("--reviewid",
required=True,
help="review id to post to")
parser.add_argument("--cfg", required=True, help="Configuration file")
parser.add_argument("--out",
required=True,
help="Output file location (e.g. patch.diff)")
args = parser.parse_args()
reviewid = args.reviewid
config = ConfigParser.ConfigParser()
try:
config.read(args.cfg)
client = RBClient(config.get('jrbb', 'reviewboard_server'),
username=config.get('jrbb', 'reviewboard_user'),
api_token=config.get('jrbb', 'reviewboard_apitoken'))
except ConfigParser.NoSectionError:
print "Configuration file " + args.cfg + " not found or missing items"
exit(1)
root = client.get_root()
# Get the revision number of the latest diff from the review
# pylint: disable=no-member
reviewrequest = root.get_review_request(review_request_id=reviewid)
diffrevision = reviewrequest.get_latest_diff().revision
print "Latest diff revision for review", reviewid, "is", diffrevision
diff = root.get_diff(review_request_id=reviewid,
diff_revision=diffrevision)
patch = diff.get_patch()
print "Retrieved the following patch file"
print "-------------------------------------------------------------------"
print patch.data
print "-------------------------------------------------------------------"
outfile = open(args.out, "w")
print >> outfile, patch.data
outfile.close()
print "Patch written to " + args.out
def get_root(self, server_url):
"""Returns the root resource of an RBClient."""
cookie_file = self.get_cookie()
self.rb_api = RBClient(server_url,
cookie_file=cookie_file,
username=self.options.username,
password=self.options.password,
auth_callback=self.credentials_prompt)
root = None
try:
root = self.rb_api.get_root()
except ServerInterfaceError, e:
die("Could not reach the review board server at %s" % server_url)
def ProcessReviewRequest(payload, tool_settings):
"""Execute an automated review on a review request."""
routing_key = ProcessReviewRequest.request.delivery_info['routing_key']
route_parts = routing_key.partition('.')
tool_ep = route_parts[0]
logger.info("Request to execute review tool '%s' for %s" %
(tool_ep, payload['url']))
try:
logger.info("Initializing RB API")
api_client = RBClient(payload['url'],
cookie_file=COOKIE_FILE,
agent=AGENT,
session=payload['session'])
api_root = api_client.get_root()
except:
logger.error("Could not contact RB server at '%s'" % payload['url'])
return False
logger.info("Loading requested tool '%s'" % tool_ep)
tools = []
for ep in pkg_resources.iter_entry_points(group='reviewbot.tools',
name=tool_ep):
tools.append(ep.load())
if len(tools) > 1:
_update_tool_execution(api_root,
payload['request'],
status=FAILED,
msg="Tool '%s' is ambiguous" % tool_ep)
return False
elif len(tools) == 0:
_update_tool_execution(api_root,
payload['request'],
status=FAILED,
msg="Tool '%s' not found" % tool_ep)
return False
tool = tools[0]
try:
logger.info("Initializing review")
review = Review(api_root, payload['request'],
payload['review_settings'])
except Exception, e:
_update_tool_execution(api_root,
payload['request'],
status=FAILED,
msg="Error initializing review: %s" % str(e))
return False
def get_open_reviews(args):
# get open reviews to a specified user, group, etc.
args['status'] = 'pending'
args['max_results'] = MAX_RESULTS
client = RBClient(RB_URL)
root = client.get_root()
if not root:
print "Error - could not get RBClient root."
return False
req = root.get_review_requests(**args)
print "\n\nGot %d pending/unsubmitted reviews" % req.total_results
for review in req:
print "%d - %s - %s" % (review.id, review.get_submitter().username,
review.summary)
def update_tools_list(panel, payload):
"""Update the RB server with installed tools.
This will detect the installed analysis tool plugins
and inform Review Board of them.
"""
logging.info("Request to refresh installed tools from '%s'" %
payload['url'])
logging.info("Iterating Tools")
tools = []
for ep in pkg_resources.iter_entry_points(group='reviewbot.tools'):
entry_point = ep.name
tool_class = ep.load()
tool = tool_class()
logging.info("Tool: %s" % entry_point)
if tool.check_dependencies():
tools.append({
'name': tool_class.name,
'entry_point': entry_point,
'version': tool_class.version,
'description': tool_class.description,
'tool_options': json.dumps(tool_class.options),
})
else:
logging.warning("%s dependency check failed." % ep.name)
logging.info("Done iterating Tools")
tools = json.dumps(tools)
hostname = panel.hostname
try:
api_client = RBClient(
payload['url'],
cookie_file=COOKIE_FILE,
agent=AGENT,
session=payload['session'])
api_root = api_client.get_root()
except Exception, e:
logging.error("Could not reach RB server: %s" % str(e))
return {'error': 'Could not reach RB server.'}
def _make_api_client(self, server_url):
"""Return an RBClient object for the server.
The RBClient will be instantiated with the proper arguments
for talking to the provided Review Board server url.
"""
return RBClient(server_url,
username=self.options.username,
password=self.options.password,
api_token=self.options.api_token,
auth_callback=self.credentials_prompt,
otp_token_callback=self.otp_token_prompt,
disable_proxy=not self.options.enable_proxy,
verify_ssl=not self.options.disable_ssl_verification,
allow_caching=not self.options.disable_cache,
cache_location=self.options.cache_location,
in_memory_cache=self.options.in_memory_cache,
save_cookies=self.options.save_cookies,
ext_auth_cookies=self.options.ext_auth_cookies)
def get_open_reviews(args):
"""
get open reviews to a specified user, group, etc.
"""
args['status'] = 'pending'
if 'max_results' not in args:
args['max_results'] = 100
client = RBClient(RB_URL)
root = client.get_root()
if not root:
print "Error - could not get RBClient root."
return False
req = root.get_review_requests(**args)
ret = {'total': req.total_results, 'reviews': []}
for review in req:
ret['reviews'].append("(%s) %s <%s/r/%d/>" %
(review.get_submitter().username, review.summary,
RB_URL, review.id))
return ret
def fetch_repositories(url, user=None, token=None):
"""Fetch repositories from Review Board.
Args:
url (unicode):
The configured url for the connection.
user (unicode):
The configured user for the connection.
token (unicode):
The configured API token for the user.
"""
logging.info('Fetching repositories from Review Board: %s', url)
# TODO: merge with COOKIE_FILE/AGENT in tasks.py
root = RBClient(url, username=user, api_token=token,
cookie_file='reviewbot-cookies.txt',
agent='ReviewBot').get_root()
for tool_type in ('Mercurial', 'Git'):
repos = root.get_repositories(tool=tool_type, only_links='',
only_fields='path,mirror_path,name')
for repo in repos.all_items:
repo_source = None
for path in (repo.path, repo.mirror_path):
if (os.path.exists(path) or path.startswith('http') or
path.startswith('git')):
repo_source = path
break
if repo_source:
init_repository(repo.name, tool_type.lower(), repo_source)
else:
logging.warn('Cannot find usable path for repository: %s',
repo.name)
def process_review_requests(client, channel, nick, review_ids):
"""
Processes a list of review request ids using a shared client
"""
logger.info("Starting codereview of: {0}".format(review_ids))
api = RBClient(settings.CODEREVIEW_REVIEWBOARD_API_URL,
username=settings.CODEREVIEW_REVIEWBOARD_API_USERNAME,
password=settings.CODEREVIEW_REVIEWBOARD_API_PASSWORD)
try:
api_root = api.get_root()
except:
logger.exception("Cannot access reviewboard")
client.msg(
channel,
"I can't complete your review {0} because I can't access reviewboard"
.format(nick))
return
errors = []
for review_id in review_ids:
try:
do_review(client, channel, nick, api_root, review_id)
except:
logger.exception("Cannot codereview cr{0}".format(review_id))
errors.append(review_id)
if errors:
cr_list = ', '.join(map(lambda id: 'cr{0}'.format(id), errors))
client.msg(
channel,
'Codereview complete {0}, but I was unable to review: {1}'.format(
nick, cr_list))
else:
client.msg(channel, 'Codereview complete {0}'.format(nick))
def main(url, group_names, days_old=7, dry_run=False):
""" do something """
try:
user = os.environ['RBUSER']
except KeyError:
raise SystemExit(
"please set RBUSER environment variable for reviewboard user")
try:
passwd = os.environ['RBPASS']
except KeyError:
raise SystemExit(
"please set RBPASS environment variable for reviewboard password")
#client = RBClient(url, user, passwd)
client = RBClient(url)
root = client.get_root()
if not root:
raise SystemExit("Error - could not get RBClient root.")
for g_name in group_names:
o = get_group_id_by_name(root, g_name, dry_run=dry_run)
if not o:
raise SystemExit("ERROR: no group '%s' found." % g_name)
logger.debug("Found group '%s' id=%d" % (g_name, o))
reviews = get_reviews_for_groups(root, group_names, dry_run=dry_run)
old_reviews = filter_reviews_older_than(root,
reviews,
days_old,
dry_run=dry_run)
logger.info(
"found %d reviews for target groups and last updated %d or more days ago"
% (len(old_reviews), days_old))
if len(old_reviews) < 1:
logger.info("Found no reviews matching criteria, exiting")
return False
users = get_submitters_for_reviews(old_reviews)
logger.debug("got user information for %d users" % len(users))
recipients = []
for u in users:
recipients.append("{u.fullname} <{u.email}>".format(u=users[u]))
table = generate_report_html_table(old_reviews, url)
body = "<h1>ReviewBoard reminder</h1>\n"
body += """<p>You're receiving this message because you have one or more pending code reviews on <a href="{url}">{url}</a>
targeted at the '{group_names}' group(s) that have not been updated in over {days_old} days and have not been submitted.
At your convenience, please evaluate these reviews and close/submit any that have been merged or discarded.
Thank You.</p>\n""".format(url=url,
days_old=days_old,
group_names=", ".join(group_names))
body += table
body += "\n<br />\n"
host = node()
user = getuser()
body += """
<p><em>generated by <a href=\"https://github.com/jantman/misc-scripts/blob/master/reviewboard_reminder_mail.py">reviewboard_reminder_mail.py</a>
running on {host} as {user} at {ds}</em></p>
""".format(host=host, user=user, ds=datetime.datetime.now().isoformat())
if dry_run:
print(
"Message to send:\n##############################\n{msg}\n#################################\n"
.format(msg=body))
print("Would send to:\n {to}".format(to=", ".join(recipients)))
else:
raise SystemExit(
"Oops - never actually implemented the mail sending...")
return True
print(
"ERROR: You must specify a reviewboard server URL (-u|--url) to use"
)
sys.exit(2)
if not options.repo:
print("ERROR: You must specify a repo (-r|--repo) to find reviews for")
sys.exit(2)
if not options.branch:
print(
"ERROR: You must specify a branch (-b|--branch) to find reviews for"
)
sys.exit(2)
client = RBClient(options.url, username=RB_USER, password=RB_PASSWORD)
root = client.get_root()
if not root:
print("Error - could not get RBClient root.")
sys.exit(1)
repo = get_repository_id_by_name(root, options.repo, verbose=VERBOSE)
if repo is None:
print("ERROR: Could not find ReviewBoard repository with name '%s'" %
options.repo)
sys.exit(3)
reviews = get_reviews_for_branch(root,
repo,
options.branch,
verbose=VERBOSE)
def create_using_reviewboard_url(cls, reviewboard_url, **rb_client_kwargs):
rb_client = RBClient(reviewboard_url, **rb_client_kwargs)
return cls(rb_client)
