def project_hosting_issue_search(self, instance): auth_config = auth.extract_auth_config_from_options(self.options) authenticator = auth.get_authenticator_for_host( 'bugs.chromium.org', auth_config) http = authenticator.authorize(httplib2.Http()) url = ('https://monorail-prod.appspot.com/_ah/api/monorail/v1/projects' '/%s/issues') % instance['name'] epoch = datetime.utcfromtimestamp(0) user_str = '*****@*****.**' % self.user query_data = urllib.urlencode({ 'maxResults': 10000, 'q': user_str, 'publishedMax': '%d' % (self.modified_before - epoch).total_seconds(), 'updatedMin': '%d' % (self.modified_after - epoch).total_seconds(), }) url = url + '?' + query_data _, body = http.request(url) content = json.loads(body) if not content: logging.error('Unable to parse %s response from projecthosting.', instance['name']) return [] issues = [] if 'items' in content: items = content['items'] for item in items: issue = { 'header': item['title'], 'created': dateutil.parser.parse(item['published']), 'modified': dateutil.parser.parse(item['updated']), 'author': item['author']['name'], 'url': 'https://code.google.com/p/%s/issues/detail?id=%s' % (instance['name'], item['id']), 'comments': [], 'status': item['status'], } if 'shorturl' in instance: issue['url'] = 'http://%s/%d' % (instance['shorturl'], item['id']) if 'owner' in item: issue['owner'] = item['owner']['name'] else: issue['owner'] = 'None' if issue['owner'] == user_str or issue['author'] == user_str: issues.append(issue) return issues
def main(argv): parser = argparse.ArgumentParser() parser.add_argument("-v", "--verbose", action="store_true") subparsers = parser.add_subparsers(dest="command") put_parser = subparsers.add_parser("put") put_parser.add_argument( "-b", "--bucket", help=( "The bucket to schedule the build on. Typically the master name, e.g." " master.tryserver.chromium.linux." ), required=True, ) put_parser.add_argument("-c", "--changes", help="A flie to load a JSON list of changes dicts from.") put_parser.add_argument("-n", "--builder-name", help="The builder to schedule the build on.", required=True) put_parser.add_argument("-p", "--properties", help="A file to load a JSON dict of properties from.") args = parser.parse_args() # TODO(smut): When more commands are implemented, refactor this. assert args.command == "put" changes = [] if args.changes: try: with open(args.changes) as fp: changes.extend(json.load(fp)) except (TypeError, ValueError): sys.stderr.write("%s contained invalid JSON list.\n" % args.changes) raise properties = {} if args.properties: try: with open(args.properties) as fp: properties.update(json.load(fp)) except (TypeError, ValueError): sys.stderr.write("%s contained invalid JSON dict.\n" % args.properties) raise authenticator = auth.get_authenticator_for_host(BUILDBUCKET_URL, auth.make_auth_config(use_oauth2=True)) http = authenticator.authorize(httplib2.Http()) http.force_exception_to_status_code = True response, content = http.request( PUT_BUILD_URL, "PUT", body=json.dumps( { "bucket": args.bucket, "parameters_json": json.dumps( {"builder_name": args.builder_name, "changes": changes, "properties": properties} ), } ), headers={"Content-Type": "application/json"}, ) if args.verbose: print content return response.status != 200
def monorail_get_auth_http(self): auth_config = auth.extract_auth_config_from_options(self.options) authenticator = auth.get_authenticator_for_host( 'bugs.chromium.org', auth_config) # Manually use a long timeout (10m); for some users who have a # long history on the issue tracker, whatever the default timeout # is is reached. return authenticator.authorize(httplib2.Http(timeout=600))
def parse_args(self, args=None, values=None): """Parses options and returns (hostname, auth.Authenticator object).""" options, args = optparse.OptionParser.parse_args(self, args, values) levels = [logging.WARNING, logging.INFO, logging.DEBUG] logging.basicConfig(level=levels[min(options.verbose, len(levels) - 1)]) auth_config = auth.extract_auth_config_from_options(options) if len(args) != 1: self.error("Expecting single argument (hostname).") if not auth_config.use_oauth2: self.error("This command is only usable with OAuth2 authentication") return args[0], auth.get_authenticator_for_host(args[0], auth_config)
def project_hosting_issue_search(self, instance): auth_config = auth.extract_auth_config_from_options(self.options) authenticator = auth.get_authenticator_for_host( "code.google.com", auth_config) http = authenticator.authorize(httplib2.Http()) url = "https://www.googleapis.com/projecthosting/v2/projects/%s/issues" % ( instance["name"]) epoch = datetime.utcfromtimestamp(0) user_str = '*****@*****.**' % self.user query_data = urllib.urlencode({ 'maxResults': 10000, 'q': user_str, 'publishedMax': '%d' % (self.modified_before - epoch).total_seconds(), 'updatedMin': '%d' % (self.modified_after - epoch).total_seconds(), }) url = url + '?' + query_data _, body = http.request(url) content = json.loads(body) if not content: print "Unable to parse %s response from projecthosting." % ( instance["name"]) return [] issues = [] if 'items' in content: items = content['items'] for item in items: issue = { "header": item["title"], "created": item["published"], "modified": item["updated"], "author": item["author"]["name"], "url": "https://code.google.com/p/%s/issues/detail?id=%s" % (instance["name"], item["id"]), "comments": [] } if 'owner' in item: issue['owner'] = item['owner']['name'] else: issue['owner'] = 'None' if issue['owner'] == user_str or issue['author'] == user_str: issues.append(issue) return issues
def parse_args(self, args=None, values=None): """Parses options and returns (hostname, auth.Authenticator object).""" options, args = optparse.OptionParser.parse_args(self, args, values) levels = [logging.WARNING, logging.INFO, logging.DEBUG] logging.basicConfig(level=levels[min(options.verbose, len(levels) - 1)]) auth_config = auth.extract_auth_config_from_options(options) if len(args) != 1: self.error('Expecting single argument (hostname).') if not auth_config.use_oauth2: self.error( 'This command is only usable with OAuth2 authentication') return args[0], auth.get_authenticator_for_host(args[0], auth_config)
def project_hosting_issue_search(self, instance): auth_config = auth.extract_auth_config_from_options(self.options) authenticator = auth.get_authenticator_for_host( "code.google.com", auth_config) http = authenticator.authorize(httplib2.Http()) url = "https://www.googleapis.com/projecthosting/v2/projects/%s/issues" % ( instance["name"]) epoch = datetime.utcfromtimestamp(0) user_str = '*****@*****.**' % self.user query_data = urllib.urlencode({ 'maxResults': 10000, 'q': user_str, 'publishedMax': '%d' % (self.modified_before - epoch).total_seconds(), 'updatedMin': '%d' % (self.modified_after - epoch).total_seconds(), }) url = url + '?' + query_data _, body = http.request(url) content = json.loads(body) if not content: print "Unable to parse %s response from projecthosting." % ( instance["name"]) return [] issues = [] if 'items' in content: items = content['items'] for item in items: issue = { "header": item["title"], "created": item["published"], "modified": item["updated"], "author": item["author"]["name"], "url": "https://code.google.com/p/%s/issues/detail?id=%s" % ( instance["name"], item["id"]), "comments": [] } if 'owner' in item: issue['owner'] = item['owner']['name'] else: issue['owner'] = 'None' if issue['owner'] == user_str or issue['author'] == user_str: issues.append(issue) return issues
def main(argv): parser = argparse.ArgumentParser() parser.add_argument( '-v', '--verbose', action='store_true', ) subparsers = parser.add_subparsers(dest='command') get_parser = subparsers.add_parser('get') get_parser.add_argument( '--id', help='The ID of the build to get the status of.', required=True, ) put_parser = subparsers.add_parser('put') put_parser.add_argument( '-b', '--bucket', help=( 'The bucket to schedule the build on. Typically the master name, e.g.' ' master.tryserver.chromium.linux.' ), required=True, ) put_parser.add_argument( '-c', '--changes', help='A flie to load a JSON list of changes dicts from.', ) put_parser.add_argument( '-n', '--builder-name', help='The builder to schedule the build on.', required=True, ) put_parser.add_argument( '-p', '--properties', help='A file to load a JSON dict of properties from.', ) args = parser.parse_args() body = None if args.command == 'get': method = 'GET' url = '%s/%s' % (BUILDBUCKET_API_URL, args.id) elif args.command == 'put': changes = [] if args.changes: try: with open(args.changes) as fp: changes.extend(json.load(fp)) except (TypeError, ValueError): sys.stderr.write('%s contained invalid JSON list.\n' % args.changes) raise properties = {} if args.properties: try: with open(args.properties) as fp: properties.update(json.load(fp)) except (TypeError, ValueError): sys.stderr.write('%s contained invalid JSON dict.\n' % args.properties) raise body = json.dumps({ 'bucket': args.bucket, 'parameters_json': json.dumps({ 'builder_name': args.builder_name, 'changes': changes, 'properties': properties, }), }) method = 'PUT' url = BUILDBUCKET_API_URL authenticator = auth.get_authenticator_for_host( BUILDBUCKET_URL, auth.make_auth_config(use_oauth2=True), ) http = authenticator.authorize(httplib2.Http()) http.force_exception_to_status_code = True response, content = http.request( url, method, body=body, headers={'Content-Type': 'application/json'}, ) if args.verbose: print content return response.status != 200
def main(argv): parser = argparse.ArgumentParser() parser.add_argument( '-v', '--verbose', action='store_true', ) subparsers = parser.add_subparsers(dest='command') put_parser = subparsers.add_parser('put') put_parser.add_argument( '-b', '--bucket', help= ('The bucket to schedule the build on. Typically the master name, e.g.' ' master.tryserver.chromium.linux.'), required=True, ) put_parser.add_argument( '-c', '--changes', help='A flie to load a JSON list of changes dicts from.', ) put_parser.add_argument( '-n', '--builder-name', help='The builder to schedule the build on.', required=True, ) put_parser.add_argument( '-p', '--properties', help='A file to load a JSON dict of properties from.', ) args = parser.parse_args() # TODO(smut): When more commands are implemented, refactor this. assert args.command == 'put' changes = [] if args.changes: try: with open(args.changes) as fp: changes.extend(json.load(fp)) except (TypeError, ValueError): sys.stderr.write('%s contained invalid JSON list.\n' % args.changes) raise properties = {} if args.properties: try: with open(args.properties) as fp: properties.update(json.load(fp)) except (TypeError, ValueError): sys.stderr.write('%s contained invalid JSON dict.\n' % args.properties) raise authenticator = auth.get_authenticator_for_host( BUILDBUCKET_URL, auth.make_auth_config(use_oauth2=True), ) http = authenticator.authorize(httplib2.Http()) http.force_exception_to_status_code = True response, content = http.request( PUT_BUILD_URL, 'PUT', body=json.dumps({ 'bucket': args.bucket, 'parameters_json': json.dumps({ 'builder_name': args.builder_name, 'changes': changes, 'properties': properties, }), }), headers={'Content-Type': 'application/json'}, ) if args.verbose: print content return response.status != 200
def monorail_get_auth_http(self): auth_config = auth.extract_auth_config_from_options(self.options) authenticator = auth.get_authenticator_for_host( 'bugs.chromium.org', auth_config) return authenticator.authorize(httplib2.Http())
def has_cookie(instance): auth_config = auth.extract_auth_config_from_options(self.options) a = auth.get_authenticator_for_host(instance['url'], auth_config) return a.has_cached_credentials()
def CheckChangedLUCIConfigs(input_api, output_api): import collections import base64 import json import urllib2 import auth import git_cl LUCI_CONFIG_HOST_NAME = 'luci-config.appspot.com' cl = git_cl.Changelist() remote, remote_branch = cl.GetRemoteBranch() if remote_branch.startswith('refs/remotes/%s/' % remote): remote_branch = remote_branch.replace( 'refs/remotes/%s/' % remote, 'refs/heads/', 1) if remote_branch.startswith('refs/remotes/branch-heads/'): remote_branch = remote_branch.replace( 'refs/remotes/branch-heads/', 'refs/branch-heads/', 1) remote_host_url = cl.GetRemoteUrl() if not remote_host_url: return [output_api.PresubmitError( 'Remote host url for git has not been defined')] remote_host_url = remote_host_url.rstrip('/') if remote_host_url.endswith('.git'): remote_host_url = remote_host_url[:-len('.git')] # authentication try: authenticator = auth.get_authenticator_for_host( LUCI_CONFIG_HOST_NAME, auth.make_auth_config()) acc_tkn = authenticator.get_access_token() except auth.AuthenticationError as e: return [output_api.PresubmitError( 'Error in authenticating user.', long_text=str(e))] def request(endpoint, body=None): api_url = ('https://%s/_ah/api/config/v1/%s' % (LUCI_CONFIG_HOST_NAME, endpoint)) req = urllib2.Request(api_url) req.add_header('Authorization', 'Bearer %s' % acc_tkn.token) if body is not None: req.add_header('Content-Type', 'application/json') req.add_data(json.dumps(body)) return json.load(urllib2.urlopen(req)) try: config_sets = request('config-sets').get('config_sets') except urllib2.HTTPError as e: return [output_api.PresubmitError( 'Config set request to luci-config failed', long_text=str(e))] if not config_sets: return [output_api.PresubmitWarning('No config_sets were returned')] loc_pref = '%s/+/%s/' % (remote_host_url, remote_branch) dir_to_config_set = { '%s/' % cs['location'][len(loc_pref):].rstrip('/'): cs['config_set'] for cs in config_sets if cs['location'].startswith(loc_pref) or ('%s/' % cs['location']) == loc_pref } cs_to_files = collections.defaultdict(list) for f in input_api.AffectedFiles(): # windows file_path = f.LocalPath().replace(_os.sep, '/') for dr, cs in dir_to_config_set.iteritems(): if dr == '/' or file_path.startswith(dr): cs_to_files[cs].append({ 'path': file_path[len(dr):] if dr != '/' else file_path, 'content': base64.b64encode( '\n'.join(f.NewContents()).encode('utf-8')) }) outputs = [] for cs, f in cs_to_files.iteritems(): try: # TODO(myjang): parallelize res = request( 'validate-config', body={'config_set': cs, 'files': f}) except urllib2.HTTPError as e: return [output_api.PresubmitError( 'Validation request to luci-config failed', long_text=str(e))] for msg in res.get('messages', []): sev = msg['severity'] if sev == 'WARNING': out_f = output_api.PresubmitPromptWarning elif sev == 'ERROR' or sev == 'CRITICAL': out_f = output_api.PresubmitError else: out_f = output_api.PresubmitNotifyResult outputs.append(out_f('Config validation: %s' % msg['text'])) return outputs
def main(argv): parser = argparse.ArgumentParser() parser.add_argument( '-v', '--verbose', action='store_true', ) subparsers = parser.add_subparsers(dest='command') get_parser = subparsers.add_parser('get') get_parser.add_argument( '--id', help='The ID of the build to get the status of.', required=True, ) put_parser = subparsers.add_parser('put') put_parser.add_argument( '-b', '--bucket', help= ('The bucket to schedule the build on. Typically the master name, e.g.' ' master.tryserver.chromium.linux.'), required=True, ) put_parser.add_argument( '-c', '--changes', help='A flie to load a JSON list of changes dicts from.', ) put_parser.add_argument( '-n', '--builder-name', help='The builder to schedule the build on.', required=True, ) put_parser.add_argument( '-p', '--properties', help= ('A file to load a JSON dict of properties from. Use "-" to pipe JSON ' 'from another command.'), ) args = parser.parse_args() body = None if args.command == 'get': method = 'GET' url = '%s/%s' % (BUILDBUCKET_API_URL, args.id) elif args.command == 'put': changes = [] if args.changes: try: with open(args.changes) as fp: changes.extend(json.load(fp)) except (TypeError, ValueError): sys.stderr.write('%s contained invalid JSON list.\n' % args.changes) raise properties = {} if args.properties: try: # Allow using pipes to stream properties from another command, e.g. # echo '{"foo": "bar", "baz": 42}' | buildbucket.py -p - if args.properties == '-': properties.update(json.load(sys.stdin)) else: with open(args.properties) as fp: properties.update(json.load(fp)) except (TypeError, ValueError): sys.stderr.write('%s contained invalid JSON dict.\n' % args.properties) raise body = json.dumps({ 'bucket': args.bucket, 'parameters_json': json.dumps({ 'builder_name': args.builder_name, 'changes': changes, 'properties': properties, }), }) method = 'PUT' url = BUILDBUCKET_API_URL authenticator = auth.get_authenticator_for_host( BUILDBUCKET_URL, auth.make_auth_config(use_oauth2=True), ) http = authenticator.authorize(httplib2.Http()) http.force_exception_to_status_code = True response, content = http.request( url, method, body=body, headers={'Content-Type': 'application/json'}, ) if args.verbose: print content build_url = json.loads(content).get('build', {}).get('url') if build_url: print 'Build triggered on: %s' % build_url return response.status != 200
def CheckChangedLUCIConfigs(input_api, output_api): import collections import base64 import json import logging import urllib2 import auth import git_cl LUCI_CONFIG_HOST_NAME = 'luci-config.appspot.com' cl = git_cl.Changelist() if input_api.change.issue and input_api.gerrit: remote_branch = input_api.gerrit.GetDestRef(input_api.change.issue) else: remote, remote_branch = cl.GetRemoteBranch() if remote_branch.startswith('refs/remotes/%s/' % remote): remote_branch = remote_branch.replace( 'refs/remotes/%s/' % remote, 'refs/heads/', 1) if remote_branch.startswith('refs/remotes/branch-heads/'): remote_branch = remote_branch.replace( 'refs/remotes/branch-heads/', 'refs/branch-heads/', 1) remote_host_url = cl.GetRemoteUrl() if not remote_host_url: return [output_api.PresubmitError( 'Remote host url for git has not been defined')] remote_host_url = remote_host_url.rstrip('/') if remote_host_url.endswith('.git'): remote_host_url = remote_host_url[:-len('.git')] # authentication try: authenticator = auth.get_authenticator_for_host( LUCI_CONFIG_HOST_NAME, auth.make_auth_config()) acc_tkn = authenticator.get_access_token() except auth.AuthenticationError as e: return [output_api.PresubmitError( 'Error in authenticating user.', long_text=str(e))] def request(endpoint, body=None): api_url = ('https://%s/_ah/api/config/v1/%s' % (LUCI_CONFIG_HOST_NAME, endpoint)) req = urllib2.Request(api_url) req.add_header('Authorization', 'Bearer %s' % acc_tkn.token) if body is not None: req.add_header('Content-Type', 'application/json') req.add_data(json.dumps(body)) return json.load(urllib2.urlopen(req)) try: config_sets = request('config-sets').get('config_sets') except urllib2.HTTPError as e: return [output_api.PresubmitError( 'Config set request to luci-config failed', long_text=str(e))] if not config_sets: return [output_api.PresubmitWarning('No config_sets were returned')] loc_pref = '%s/+/%s/' % (remote_host_url, remote_branch) logging.debug('Derived location prefix: %s', loc_pref) dir_to_config_set = { '%s/' % cs['location'][len(loc_pref):].rstrip('/'): cs['config_set'] for cs in config_sets if cs['location'].startswith(loc_pref) or ('%s/' % cs['location']) == loc_pref } cs_to_files = collections.defaultdict(list) for f in input_api.AffectedFiles(): # windows file_path = f.LocalPath().replace(_os.sep, '/') logging.debug('Affected file path: %s', file_path) for dr, cs in dir_to_config_set.iteritems(): if dr == '/' or file_path.startswith(dr): cs_to_files[cs].append({ 'path': file_path[len(dr):] if dr != '/' else file_path, 'content': base64.b64encode( '\n'.join(f.NewContents()).encode('utf-8')) }) outputs = [] for cs, f in cs_to_files.iteritems(): try: # TODO(myjang): parallelize res = request( 'validate-config', body={'config_set': cs, 'files': f}) except urllib2.HTTPError as e: return [output_api.PresubmitError( 'Validation request to luci-config failed', long_text=str(e))] for msg in res.get('messages', []): sev = msg['severity'] if sev == 'WARNING': out_f = output_api.PresubmitPromptWarning elif sev == 'ERROR' or sev == 'CRITICAL': out_f = output_api.PresubmitError else: out_f = output_api.PresubmitNotifyResult outputs.append(out_f('Config validation: %s' % msg['text'])) return outputs