def new_parse_args(args=None, values=None): options, args = old_parse_args(args, values) auth_config = auth.extract_auth_config_from_options(options) if not options.issue: parser.error('Require --issue') obj = rietveld.Rietveld(options.server, auth_config, options.user) return options, args, obj
def main(): usage = 'Usage: %prog project_name author path_to_chromium' # The default HelpFormatter causes the docstring to display improperly. class VanillaHelpFormatter(optparse.IndentedHelpFormatter): def format_description(self, description): if description: return description else: return '' parser = optparse.OptionParser(usage=usage, description=sys.modules[__name__].__doc__, formatter=VanillaHelpFormatter()) parser.add_option('--no-try', action='store_true', dest='notry', help='Create the CL with NOTRY=true') auth.add_auth_options(parser) options, args = parser.parse_args() auth_config = auth.extract_auth_config_from_options(options) if len(args) != 3: parser.print_usage() return 1 AutoRoller(*args, auth_config=auth_config, options=options).main()
def monorail_get_auth_http(self): auth_config = auth.extract_auth_config_from_options(self.options) authenticator = auth.get_authenticator(auth_config) # Manually use a long timeout (10m); for some users who have a # long history on the issue tracker, whatever the default timeout # is is reached. return authenticator.authorize(httplib2.Http(timeout=600))
def main(args): """Runs cpplint on the current changelist.""" """Adapted from git_cl.py CMDlint """ parser = git_cl.OptionParser() parser.add_option('--filter', action='append', metavar='-x,+y', help='Comma-separated list of cpplint\'s category-filters') parser.add_option('--project_root') parser.add_option('--base_branch') auth.add_auth_options(parser) options, args = parser.parse_args(args) auth_config = auth.extract_auth_config_from_options(options) # Access to a protected member _XX of a client class # pylint: disable=protected-access try: import cpplint import cpplint_chromium except ImportError: print('Your depot_tools is missing cpplint.py and/or cpplint_chromium.py.') return 1 # Change the current working directory before calling lint so that it # shows the correct base. settings = git_cl.settings previous_cwd = os.getcwd() os.chdir(settings.GetRoot()) try: cl = git_cl.Changelist(auth_config=auth_config) change = cl.GetChange(git_common.get_or_create_merge_base(cl.GetBranch(), options.base_branch), None) files = [f.LocalPath() for f in change.AffectedFiles()] if not files: print('Cannot lint an empty CL') return 0 # Process cpplints arguments if any. command = args + files if options.filter: command = ['--filter=' + ','.join(options.filter)] + command if options.project_root: command = ['--project_root=' + options.project_root] + command filenames = cpplint.ParseArguments(command) white_regex = re.compile(settings.GetLintRegex()) black_regex = re.compile(settings.GetLintIgnoreRegex()) extra_check_functions = [cpplint_chromium.CheckPointerDeclarationWhitespace] for filename in filenames: if white_regex.match(filename): if black_regex.match(filename): print('Ignoring file %s' % filename) else: cpplint.ProcessFile(filename, cpplint._cpplint_state.verbose_level, extra_check_functions) else: print('Skipping file %s' % filename) finally: os.chdir(previous_cwd) print('Total errors found: %d\n' % cpplint._cpplint_state.error_count) if cpplint._cpplint_state.error_count != 0: return 1 return 0
def project_hosting_issue_search(self, instance): auth_config = auth.extract_auth_config_from_options(self.options) authenticator = auth.get_authenticator_for_host( 'bugs.chromium.org', auth_config) http = authenticator.authorize(httplib2.Http()) url = ('https://monorail-prod.appspot.com/_ah/api/monorail/v1/projects' '/%s/issues') % instance['name'] epoch = datetime.utcfromtimestamp(0) user_str = '*****@*****.**' % self.user query_data = urllib.urlencode({ 'maxResults': 10000, 'q': user_str, 'publishedMax': '%d' % (self.modified_before - epoch).total_seconds(), 'updatedMin': '%d' % (self.modified_after - epoch).total_seconds(), }) url = url + '?' + query_data _, body = http.request(url) content = json.loads(body) if not content: logging.error('Unable to parse %s response from projecthosting.', instance['name']) return [] issues = [] if 'items' in content: items = content['items'] for item in items: issue = { 'header': item['title'], 'created': dateutil.parser.parse(item['published']), 'modified': dateutil.parser.parse(item['updated']), 'author': item['author']['name'], 'url': 'https://code.google.com/p/%s/issues/detail?id=%s' % (instance['name'], item['id']), 'comments': [], 'status': item['status'], } if 'shorturl' in instance: issue['url'] = 'http://%s/%d' % (instance['shorturl'], item['id']) if 'owner' in item: issue['owner'] = item['owner']['name'] else: issue['owner'] = 'None' if issue['owner'] == user_str or issue['author'] == user_str: issues.append(issue) return issues
def monorail_get_auth_http(self): auth_config = auth.extract_auth_config_from_options(self.options) authenticator = auth.get_authenticator_for_host( 'bugs.chromium.org', auth_config) # Manually use a long timeout (10m); for some users who have a # long history on the issue tracker, whatever the default timeout # is is reached. return authenticator.authorize(httplib2.Http(timeout=600))
def parse_args(self, args=None, values=None): """Parses options and returns (hostname, auth.Authenticator object).""" options, args = optparse.OptionParser.parse_args(self, args, values) levels = [logging.WARNING, logging.INFO, logging.DEBUG] logging.basicConfig(level=levels[min(options.verbose, len(levels) - 1)]) auth_config = auth.extract_auth_config_from_options(options) if len(args) != 1: self.error("Expecting single argument (hostname).") if not auth_config.use_oauth2: self.error("This command is only usable with OAuth2 authentication") return args[0], auth.get_authenticator_for_host(args[0], auth_config)
def project_hosting_issue_search(self, instance): auth_config = auth.extract_auth_config_from_options(self.options) authenticator = auth.get_authenticator_for_host( "code.google.com", auth_config) http = authenticator.authorize(httplib2.Http()) url = "https://www.googleapis.com/projecthosting/v2/projects/%s/issues" % ( instance["name"]) epoch = datetime.utcfromtimestamp(0) user_str = '*****@*****.**' % self.user query_data = urllib.urlencode({ 'maxResults': 10000, 'q': user_str, 'publishedMax': '%d' % (self.modified_before - epoch).total_seconds(), 'updatedMin': '%d' % (self.modified_after - epoch).total_seconds(), }) url = url + '?' + query_data _, body = http.request(url) content = json.loads(body) if not content: print "Unable to parse %s response from projecthosting." % ( instance["name"]) return [] issues = [] if 'items' in content: items = content['items'] for item in items: issue = { "header": item["title"], "created": item["published"], "modified": item["updated"], "author": item["author"]["name"], "url": "https://code.google.com/p/%s/issues/detail?id=%s" % (instance["name"], item["id"]), "comments": [] } if 'owner' in item: issue['owner'] = item['owner']['name'] else: issue['owner'] = 'None' if issue['owner'] == user_str or issue['author'] == user_str: issues.append(issue) return issues
def parse_args(self, args=None, values=None): """Parses options and returns (hostname, auth.Authenticator object).""" options, args = optparse.OptionParser.parse_args(self, args, values) levels = [logging.WARNING, logging.INFO, logging.DEBUG] logging.basicConfig(level=levels[min(options.verbose, len(levels) - 1)]) auth_config = auth.extract_auth_config_from_options(options) if len(args) != 1: self.error('Expecting single argument (hostname).') if not auth_config.use_oauth2: self.error( 'This command is only usable with OAuth2 authentication') return args[0], auth.get_authenticator_for_host(args[0], auth_config)
def rietveld_search(self, instance, owner=None, reviewer=None): if instance['requires_auth'] and not instance['auth']: return [] email = None if instance['auth'] else '' auth_config = auth.extract_auth_config_from_options(self.options) remote = rietveld.Rietveld('https://' + instance['url'], auth_config, email) # See def search() in rietveld.py to see all the filters you can use. query_modified_after = None if instance['supports_owner_modified_query']: query_modified_after = self.modified_after.strftime('%Y-%m-%d') # Rietveld does not allow search by both created_before and modified_after. # (And some instances don't allow search by both owner and modified_after) owner_email = None reviewer_email = None if owner: owner_email = owner + '@' + instance['email_domain'] if reviewer: reviewer_email = reviewer + '@' + instance['email_domain'] issues = remote.search(owner=owner_email, reviewer=reviewer_email, modified_after=query_modified_after, with_messages=True) self.show_progress() issues = filter( lambda i: (datetime_from_rietveld(i['created']) < self.modified_before), issues) issues = filter( lambda i: (datetime_from_rietveld(i['modified']) > self.modified_after), issues) should_filter_by_user = True issues = map(partial(self.process_rietveld_issue, remote, instance), issues) issues = filter( partial(self.filter_issue, should_filter_by_user=should_filter_by_user), issues) issues = sorted(issues, key=lambda i: i['modified'], reverse=True) return issues
def project_hosting_issue_search(self, instance): auth_config = auth.extract_auth_config_from_options(self.options) authenticator = auth.get_authenticator_for_host( "code.google.com", auth_config) http = authenticator.authorize(httplib2.Http()) url = "https://www.googleapis.com/projecthosting/v2/projects/%s/issues" % ( instance["name"]) epoch = datetime.utcfromtimestamp(0) user_str = '*****@*****.**' % self.user query_data = urllib.urlencode({ 'maxResults': 10000, 'q': user_str, 'publishedMax': '%d' % (self.modified_before - epoch).total_seconds(), 'updatedMin': '%d' % (self.modified_after - epoch).total_seconds(), }) url = url + '?' + query_data _, body = http.request(url) content = json.loads(body) if not content: print "Unable to parse %s response from projecthosting." % ( instance["name"]) return [] issues = [] if 'items' in content: items = content['items'] for item in items: issue = { "header": item["title"], "created": item["published"], "modified": item["updated"], "author": item["author"]["name"], "url": "https://code.google.com/p/%s/issues/detail?id=%s" % ( instance["name"], item["id"]), "comments": [] } if 'owner' in item: issue['owner'] = item['owner']['name'] else: issue['owner'] = 'None' if issue['owner'] == user_str or issue['author'] == user_str: issues.append(issue) return issues
def rietveld_search(self, instance, owner=None, reviewer=None): if instance['requires_auth'] and not instance['auth']: return [] email = None if instance['auth'] else '' auth_config = auth.extract_auth_config_from_options(self.options) remote = rietveld.Rietveld('https://' + instance['url'], auth_config, email) # See def search() in rietveld.py to see all the filters you can use. query_modified_after = None if instance['supports_owner_modified_query']: query_modified_after = self.modified_after.strftime('%Y-%m-%d') # Rietveld does not allow search by both created_before and modified_after. # (And some instances don't allow search by both owner and modified_after) owner_email = None reviewer_email = None if owner: owner_email = owner + '@' + instance['email_domain'] if reviewer: reviewer_email = reviewer + '@' + instance['email_domain'] issues = remote.search( owner=owner_email, reviewer=reviewer_email, modified_after=query_modified_after, with_messages=True) self.show_progress() issues = filter( lambda i: (datetime_from_rietveld(i['created']) < self.modified_before), issues) issues = filter( lambda i: (datetime_from_rietveld(i['modified']) > self.modified_after), issues) should_filter_by_user = True issues = map(partial(self.process_rietveld_issue, remote, instance), issues) issues = filter( partial(self.filter_issue, should_filter_by_user=should_filter_by_user), issues) issues = sorted(issues, key=lambda i: i['modified'], reverse=True) return issues
def main(): # TODO(pgervais,tandrii): split this func, it's still too long. sys.stdout = Unbuffered(sys.stdout) parser = _get_arg_parser() options, args = parser.parse_args() auth_config = auth.extract_auth_config_from_options(options) if options.whitelist and options.blacklist: parser.error('Cannot specify both --whitelist and --blacklist') if options.email and options.email_file: parser.error('-e and -E options are incompatible') if (os.path.isfile(os.path.join(os.getcwd(), 'update.flag')) and not options.force): print 'update.flag file found: bot_update has run and checkout is already ' print 'in a consistent state. No actions will be performed in this step.' return 0 logging.basicConfig( format='%(levelname)5s %(module)11s(%(lineno)4d): %(message)s', level=[logging.WARNING, logging.INFO, logging.DEBUG][min(2, options.verbose)]) if args: parser.error('Extra argument(s) "%s" not understood' % ' '.join(args)) if not options.issue: parser.error('Require --issue') options.server = options.server.rstrip('/') if not options.server: parser.error('Require a valid server') options.revision_mapping = json.loads(options.revision_mapping) # read email if needed if options.email_file: if not os.path.exists(options.email_file): parser.error('file does not exist: %s' % options.email_file) with open(options.email_file, 'rb') as f: options.email = f.read().strip() print('Connecting to %s' % options.server) # Always try un-authenticated first, except for OAuth2 if options.private_key_file: # OAuth2 authentication rietveld_obj = rietveld.JwtOAuth2Rietveld(options.server, options.email, options.private_key_file) try: properties = rietveld_obj.get_issue_properties( options.issue, False) except urllib2.URLError: logging.exception('failed to fetch issue properties') sys.exit(RETURN_CODE_INFRA_FAILURE) else: # Passing None as auth_config disables authentication. rietveld_obj = rietveld.Rietveld(options.server, None) properties = None # Bad except clauses order (HTTPError is an ancestor class of # ClientLoginError) # pylint: disable=bad-except-order try: properties = rietveld_obj.get_issue_properties( options.issue, False) except urllib2.HTTPError as e: if e.getcode() != 302: raise if options.no_auth: exit('FAIL: Login detected -- is issue private?') # TODO(maruel): A few 'Invalid username or password.' are printed first, # we should get rid of those. except urllib2.URLError: logging.exception('failed to fetch issue properties') return RETURN_CODE_INFRA_FAILURE except rietveld.upload.ClientLoginError as e: # Fine, we'll do proper authentication. pass if properties is None: rietveld_obj = rietveld.Rietveld(options.server, auth_config, options.email) try: properties = rietveld_obj.get_issue_properties( options.issue, False) except rietveld.upload.ClientLoginError as e: print('Accessing the issue requires proper credentials.') return RETURN_CODE_OTHER_FAILURE except urllib2.URLError: logging.exception('failed to fetch issue properties') return RETURN_CODE_INFRA_FAILURE if not options.patchset: options.patchset = properties['patchsets'][-1] print('No patchset specified. Using patchset %d' % options.patchset) issues_patchsets_to_apply = [(options.issue, options.patchset)] try: depends_on_info = rietveld_obj.get_depends_on_patchset( options.issue, options.patchset) except urllib2.URLError: logging.exception('failed to fetch depends_on_patchset') return RETURN_CODE_INFRA_FAILURE while depends_on_info: depends_on_issue = int(depends_on_info['issue']) depends_on_patchset = int(depends_on_info['patchset']) try: depends_on_info = rietveld_obj.get_depends_on_patchset( depends_on_issue, depends_on_patchset) issues_patchsets_to_apply.insert( 0, (depends_on_issue, depends_on_patchset)) except urllib2.HTTPError: print( 'The patchset that was marked as a dependency no longer ' 'exists: %s/%d/#ps%d' % (options.server, depends_on_issue, depends_on_patchset)) print 'Therefore it is likely that this patch will not apply cleanly.' print depends_on_info = None except urllib2.URLError: logging.exception('failed to fetch dependency issue') return RETURN_CODE_INFRA_FAILURE num_issues_patchsets_to_apply = len(issues_patchsets_to_apply) if num_issues_patchsets_to_apply > 1: print print 'apply_issue.py found %d dependent CLs.' % ( num_issues_patchsets_to_apply - 1) print 'They will be applied in the following order:' num = 1 for issue_to_apply, patchset_to_apply in issues_patchsets_to_apply: print ' #%d %s/%d/#ps%d' % (num, options.server, issue_to_apply, patchset_to_apply) num += 1 print for issue_to_apply, patchset_to_apply in issues_patchsets_to_apply: issue_url = '%s/%d/#ps%d' % (options.server, issue_to_apply, patchset_to_apply) print('Downloading patch from %s' % issue_url) try: patchset = rietveld_obj.get_patch(issue_to_apply, patchset_to_apply) except urllib2.HTTPError: print( 'Failed to fetch the patch for issue %d, patchset %d.\n' 'Try visiting %s/%d') % (issue_to_apply, patchset_to_apply, options.server, issue_to_apply) # If we got this far, then this is likely missing patchset. # Thus, it's not infra failure. return RETURN_CODE_OTHER_FAILURE except urllib2.URLError: logging.exception( 'Failed to fetch the patch for issue %d, patchset %d', issue_to_apply, patchset_to_apply) return RETURN_CODE_INFRA_FAILURE if options.whitelist: patchset.patches = [ patch for patch in patchset.patches if patch.filename in options.whitelist ] if options.blacklist: patchset.patches = [ patch for patch in patchset.patches if patch.filename not in options.blacklist ] for patch in patchset.patches: print(patch) if options.extra_patchlevel: patch.patchlevel += options.extra_patchlevel full_dir = os.path.abspath(options.root_dir) scm_type = scm.determine_scm(full_dir) if scm_type == 'git': scm_obj = checkout.GitCheckout(full_dir, None, None, None, None) else: parser.error('Couldn\'t determine the scm') # TODO(maruel): HACK, remove me. # When run a build slave, make sure buildbot knows that the checkout was # modified. if options.root_dir == 'src' and getpass.getuser() == 'chrome-bot': # See sourcedirIsPatched() in: # http://src.chromium.org/viewvc/chrome/trunk/tools/build/scripts/slave/ # chromium_commands.py?view=markup open('.buildbot-patched', 'w').close() print('\nApplying the patch from %s' % issue_url) try: scm_obj.apply_patch(patchset, verbose=True) except checkout.PatchApplicationFailed as e: print(str(e)) print('CWD=%s' % os.getcwd()) print('Checkout path=%s' % scm_obj.project_path) return RETURN_CODE_OTHER_FAILURE if ('DEPS' in map(os.path.basename, patchset.filenames) and not options.ignore_deps): gclient_root = gclient_utils.FindGclientRoot(full_dir) if gclient_root and scm_type: print( 'A DEPS file was updated inside a gclient checkout, running gclient ' 'sync.') gclient_path = os.path.join(BASE_DIR, 'gclient') if sys.platform == 'win32': gclient_path += '.bat' with annotated_gclient.temp_filename(suffix='gclient') as f: cmd = [ gclient_path, 'sync', '--nohooks', '--delete_unversioned_trees', ] if options.revision_mapping: cmd.extend(['--output-json', f]) retcode = subprocess.call(cmd, cwd=gclient_root) if retcode == 0 and options.revision_mapping: revisions = annotated_gclient.parse_got_revision( f, options.revision_mapping) annotated_gclient.emit_buildprops(revisions) return retcode return RETURN_CODE_OK
def main(): # Silence upload.py. rietveld.upload.verbosity = 0 today = datetime.date.today() begin, end = get_previous_quarter(today) default_email = os.environ.get('EMAIL_ADDRESS') if not default_email: user = os.environ.get('USER') if user: default_email = user + '@chromium.org' parser = optparse.OptionParser(description=__doc__) parser.add_option('--count', action='store_true', help='Just count instead of printing individual issues') parser.add_option('-r', '--reviewer', metavar='<email>', default=default_email, help='Filter on issue reviewer, default=%default') parser.add_option('-b', '--begin', metavar='<date>', help='Filter issues created after the date') parser.add_option('-e', '--end', metavar='<date>', help='Filter issues created before the date') parser.add_option('-Q', '--last_quarter', action='store_true', help='Use last quarter\'s dates, e.g. %s to %s' % (begin, end)) parser.add_option('-i', '--instance_url', metavar='<host>', default='http://codereview.chromium.org', help='Host to use, default is %default') auth.add_auth_options(parser) # Remove description formatting parser.format_description = (lambda _: parser.description) # pylint: disable=E1101 options, args = parser.parse_args() auth_config = auth.extract_auth_config_from_options(options) if args: parser.error('Args unsupported') if options.reviewer is None: parser.error('$EMAIL_ADDRESS and $USER are not set, please use -r') print >> sys.stderr, 'Searching for reviews by %s' % options.reviewer if options.last_quarter: options.begin = begin options.end = end print >> sys.stderr, 'Using range %s to %s' % (options.begin, options.end) else: if options.begin is None or options.end is None: parser.error( 'Please specify either --last_quarter or --begin and --end') # Validate dates. try: to_datetime(options.begin) to_datetime(options.end) except ValueError as e: parser.error('%s: %s - %s' % (e, options.begin, options.end)) if options.count: print_count(options.reviewer, options.begin, options.end, options.instance_url, auth_config) else: print_reviews(options.reviewer, options.begin, options.end, options.instance_url, auth_config) return 0
def monorail_get_auth_http(self): auth_config = auth.extract_auth_config_from_options(self.options) authenticator = auth.get_authenticator_for_host( 'bugs.chromium.org', auth_config) return authenticator.authorize(httplib2.Http())
def has_cookie(instance): auth_config = auth.extract_auth_config_from_options(self.options) a = auth.get_authenticator_for_host(instance['url'], auth_config) return a.has_cached_credentials()
def main(): # TODO(pgervais): This function is way too long. Split. sys.stdout = Unbuffered(sys.stdout) parser = optparse.OptionParser(description=sys.modules[__name__].__doc__) parser.add_option( '-v', '--verbose', action='count', default=0, help='Prints debugging infos') parser.add_option( '-e', '--email', help='Email address to access rietveld. If not specified, anonymous ' 'access will be used.') parser.add_option( '-E', '--email-file', help='File containing the email address to access rietveld. ' 'If not specified, anonymous access will be used.') parser.add_option( '-k', '--private-key-file', help='Path to file containing a private key in p12 format for OAuth2 ' 'authentication with "notasecret" password (as generated by Google ' 'Cloud Console).') parser.add_option( '-i', '--issue', type='int', help='Rietveld issue number') parser.add_option( '-p', '--patchset', type='int', help='Rietveld issue\'s patchset number') parser.add_option( '-r', '--root_dir', default=os.getcwd(), help='Root directory to apply the patch') parser.add_option( '-s', '--server', default='http://codereview.chromium.org', help='Rietveld server') parser.add_option('--no-auth', action='store_true', help='Do not attempt authenticated requests.') parser.add_option('--revision-mapping', default='{}', help='When running gclient, annotate the got_revisions ' 'using the revision-mapping.') parser.add_option('-f', '--force', action='store_true', help='Really run apply_issue, even if .update.flag ' 'is detected.') parser.add_option('-b', '--base_ref', help='DEPRECATED do not use.') parser.add_option('--whitelist', action='append', default=[], help='Patch only specified file(s).') parser.add_option('--blacklist', action='append', default=[], help='Don\'t patch specified file(s).') parser.add_option('-d', '--ignore_deps', action='store_true', help='Don\'t run gclient sync on DEPS changes.') auth.add_auth_options(parser) options, args = parser.parse_args() auth_config = auth.extract_auth_config_from_options(options) if options.whitelist and options.blacklist: parser.error('Cannot specify both --whitelist and --blacklist') if options.email and options.email_file: parser.error('-e and -E options are incompatible') if (os.path.isfile(os.path.join(os.getcwd(), 'update.flag')) and not options.force): print 'update.flag file found: bot_update has run and checkout is already ' print 'in a consistent state. No actions will be performed in this step.' return 0 logging.basicConfig( format='%(levelname)5s %(module)11s(%(lineno)4d): %(message)s', level=[logging.WARNING, logging.INFO, logging.DEBUG][ min(2, options.verbose)]) if args: parser.error('Extra argument(s) "%s" not understood' % ' '.join(args)) if not options.issue: parser.error('Require --issue') options.server = options.server.rstrip('/') if not options.server: parser.error('Require a valid server') options.revision_mapping = json.loads(options.revision_mapping) # read email if needed if options.email_file: if not os.path.exists(options.email_file): parser.error('file does not exist: %s' % options.email_file) with open(options.email_file, 'rb') as f: options.email = f.read().strip() print('Connecting to %s' % options.server) # Always try un-authenticated first, except for OAuth2 if options.private_key_file: # OAuth2 authentication obj = rietveld.JwtOAuth2Rietveld(options.server, options.email, options.private_key_file) properties = obj.get_issue_properties(options.issue, False) else: # Passing None as auth_config disables authentication. obj = rietveld.Rietveld(options.server, None) properties = None # Bad except clauses order (HTTPError is an ancestor class of # ClientLoginError) # pylint: disable=E0701 try: properties = obj.get_issue_properties(options.issue, False) except urllib2.HTTPError as e: if e.getcode() != 302: raise if options.no_auth: exit('FAIL: Login detected -- is issue private?') # TODO(maruel): A few 'Invalid username or password.' are printed first, # we should get rid of those. except rietveld.upload.ClientLoginError as e: # Fine, we'll do proper authentication. pass if properties is None: obj = rietveld.Rietveld(options.server, auth_config, options.email) try: properties = obj.get_issue_properties(options.issue, False) except rietveld.upload.ClientLoginError as e: print('Accessing the issue requires proper credentials.') return 1 if not options.patchset: options.patchset = properties['patchsets'][-1] print('No patchset specified. Using patchset %d' % options.patchset) issues_patchsets_to_apply = [(options.issue, options.patchset)] depends_on_info = obj.get_depends_on_patchset(options.issue, options.patchset) while depends_on_info: depends_on_issue = int(depends_on_info['issue']) depends_on_patchset = int(depends_on_info['patchset']) try: depends_on_info = obj.get_depends_on_patchset(depends_on_issue, depends_on_patchset) issues_patchsets_to_apply.insert(0, (depends_on_issue, depends_on_patchset)) except urllib2.HTTPError: print ('The patchset that was marked as a dependency no longer ' 'exists: %s/%d/#ps%d' % ( options.server, depends_on_issue, depends_on_patchset)) print 'Therefore it is likely that this patch will not apply cleanly.' print depends_on_info = None num_issues_patchsets_to_apply = len(issues_patchsets_to_apply) if num_issues_patchsets_to_apply > 1: print print 'apply_issue.py found %d dependent CLs.' % ( num_issues_patchsets_to_apply - 1) print 'They will be applied in the following order:' num = 1 for issue_to_apply, patchset_to_apply in issues_patchsets_to_apply: print ' #%d %s/%d/#ps%d' % ( num, options.server, issue_to_apply, patchset_to_apply) num += 1 print for issue_to_apply, patchset_to_apply in issues_patchsets_to_apply: issue_url = '%s/%d/#ps%d' % (options.server, issue_to_apply, patchset_to_apply) print('Downloading patch from %s' % issue_url) try: patchset = obj.get_patch(issue_to_apply, patchset_to_apply) except urllib2.HTTPError as e: print( 'Failed to fetch the patch for issue %d, patchset %d.\n' 'Try visiting %s/%d') % ( issue_to_apply, patchset_to_apply, options.server, issue_to_apply) return 1 if options.whitelist: patchset.patches = [patch for patch in patchset.patches if patch.filename in options.whitelist] if options.blacklist: patchset.patches = [patch for patch in patchset.patches if patch.filename not in options.blacklist] for patch in patchset.patches: print(patch) full_dir = os.path.abspath(options.root_dir) scm_type = scm.determine_scm(full_dir) if scm_type == 'svn': scm_obj = checkout.SvnCheckout(full_dir, None, None, None, None) elif scm_type == 'git': scm_obj = checkout.GitCheckout(full_dir, None, None, None, None) elif scm_type == None: scm_obj = checkout.RawCheckout(full_dir, None, None) else: parser.error('Couldn\'t determine the scm') # TODO(maruel): HACK, remove me. # When run a build slave, make sure buildbot knows that the checkout was # modified. if options.root_dir == 'src' and getpass.getuser() == 'chrome-bot': # See sourcedirIsPatched() in: # http://src.chromium.org/viewvc/chrome/trunk/tools/build/scripts/slave/ # chromium_commands.py?view=markup open('.buildbot-patched', 'w').close() print('\nApplying the patch from %s' % issue_url) try: scm_obj.apply_patch(patchset, verbose=True) except checkout.PatchApplicationFailed as e: print(str(e)) print('CWD=%s' % os.getcwd()) print('Checkout path=%s' % scm_obj.project_path) return 1 if ('DEPS' in map(os.path.basename, patchset.filenames) and not options.ignore_deps): gclient_root = gclient_utils.FindGclientRoot(full_dir) if gclient_root and scm_type: print( 'A DEPS file was updated inside a gclient checkout, running gclient ' 'sync.') gclient_path = os.path.join(BASE_DIR, 'gclient') if sys.platform == 'win32': gclient_path += '.bat' with annotated_gclient.temp_filename(suffix='gclient') as f: cmd = [ gclient_path, 'sync', '--nohooks', '--delete_unversioned_trees', ] if scm_type == 'svn': cmd.extend(['--revision', 'BASE']) if options.revision_mapping: cmd.extend(['--output-json', f]) retcode = subprocess.call(cmd, cwd=gclient_root) if retcode == 0 and options.revision_mapping: revisions = annotated_gclient.parse_got_revision( f, options.revision_mapping) annotated_gclient.emit_buildprops(revisions) return retcode return 0
def main(): # TODO(pgervais): This function is way too long. Split. sys.stdout = Unbuffered(sys.stdout) parser = optparse.OptionParser(description=sys.modules[__name__].__doc__) parser.add_option('-v', '--verbose', action='count', default=0, help='Prints debugging infos') parser.add_option( '-e', '--email', help='Email address to access rietveld. If not specified, anonymous ' 'access will be used.') parser.add_option( '-E', '--email-file', help='File containing the email address to access rietveld. ' 'If not specified, anonymous access will be used.') parser.add_option( '-k', '--private-key-file', help='Path to file containing a private key in p12 format for OAuth2 ' 'authentication with "notasecret" password (as generated by Google ' 'Cloud Console).') parser.add_option('-i', '--issue', type='int', help='Rietveld issue number') parser.add_option('-p', '--patchset', type='int', help='Rietveld issue\'s patchset number') parser.add_option('-r', '--root_dir', default=os.getcwd(), help='Root directory to apply the patch') parser.add_option('-s', '--server', default='http://codereview.chromium.org', help='Rietveld server') parser.add_option('--no-auth', action='store_true', help='Do not attempt authenticated requests.') parser.add_option('--revision-mapping', default='{}', help='When running gclient, annotate the got_revisions ' 'using the revision-mapping.') parser.add_option('-f', '--force', action='store_true', help='Really run apply_issue, even if .update.flag ' 'is detected.') parser.add_option('-b', '--base_ref', help='DEPRECATED do not use.') parser.add_option('--whitelist', action='append', default=[], help='Patch only specified file(s).') parser.add_option('--blacklist', action='append', default=[], help='Don\'t patch specified file(s).') parser.add_option('-d', '--ignore_deps', action='store_true', help='Don\'t run gclient sync on DEPS changes.') auth.add_auth_options(parser) options, args = parser.parse_args() auth_config = auth.extract_auth_config_from_options(options) if options.whitelist and options.blacklist: parser.error('Cannot specify both --whitelist and --blacklist') if options.email and options.email_file: parser.error('-e and -E options are incompatible') if (os.path.isfile(os.path.join(os.getcwd(), 'update.flag')) and not options.force): print 'update.flag file found: bot_update has run and checkout is already ' print 'in a consistent state. No actions will be performed in this step.' return 0 logging.basicConfig( format='%(levelname)5s %(module)11s(%(lineno)4d): %(message)s', level=[logging.WARNING, logging.INFO, logging.DEBUG][min(2, options.verbose)]) if args: parser.error('Extra argument(s) "%s" not understood' % ' '.join(args)) if not options.issue: parser.error('Require --issue') options.server = options.server.rstrip('/') if not options.server: parser.error('Require a valid server') options.revision_mapping = json.loads(options.revision_mapping) # read email if needed if options.email_file: if not os.path.exists(options.email_file): parser.error('file does not exist: %s' % options.email_file) with open(options.email_file, 'rb') as f: options.email = f.read().strip() print('Connecting to %s' % options.server) # Always try un-authenticated first, except for OAuth2 if options.private_key_file: # OAuth2 authentication obj = rietveld.JwtOAuth2Rietveld(options.server, options.email, options.private_key_file) properties = obj.get_issue_properties(options.issue, False) else: # Passing None as auth_config disables authentication. obj = rietveld.Rietveld(options.server, None) properties = None # Bad except clauses order (HTTPError is an ancestor class of # ClientLoginError) # pylint: disable=E0701 try: properties = obj.get_issue_properties(options.issue, False) except urllib2.HTTPError as e: if e.getcode() != 302: raise if options.no_auth: exit('FAIL: Login detected -- is issue private?') # TODO(maruel): A few 'Invalid username or password.' are printed first, # we should get rid of those. except rietveld.upload.ClientLoginError as e: # Fine, we'll do proper authentication. pass if properties is None: obj = rietveld.Rietveld(options.server, auth_config, options.email) try: properties = obj.get_issue_properties(options.issue, False) except rietveld.upload.ClientLoginError as e: print('Accessing the issue requires proper credentials.') return 1 if not options.patchset: options.patchset = properties['patchsets'][-1] print('No patchset specified. Using patchset %d' % options.patchset) issues_patchsets_to_apply = [(options.issue, options.patchset)] depends_on_info = obj.get_depends_on_patchset(options.issue, options.patchset) while depends_on_info: depends_on_issue = int(depends_on_info['issue']) depends_on_patchset = int(depends_on_info['patchset']) try: depends_on_info = obj.get_depends_on_patchset( depends_on_issue, depends_on_patchset) issues_patchsets_to_apply.insert( 0, (depends_on_issue, depends_on_patchset)) except urllib2.HTTPError: print( 'The patchset that was marked as a dependency no longer ' 'exists: %s/%d/#ps%d' % (options.server, depends_on_issue, depends_on_patchset)) print 'Therefore it is likely that this patch will not apply cleanly.' print depends_on_info = None num_issues_patchsets_to_apply = len(issues_patchsets_to_apply) if num_issues_patchsets_to_apply > 1: print print 'apply_issue.py found %d dependent CLs.' % ( num_issues_patchsets_to_apply - 1) print 'They will be applied in the following order:' num = 1 for issue_to_apply, patchset_to_apply in issues_patchsets_to_apply: print ' #%d %s/%d/#ps%d' % (num, options.server, issue_to_apply, patchset_to_apply) num += 1 print for issue_to_apply, patchset_to_apply in issues_patchsets_to_apply: issue_url = '%s/%d/#ps%d' % (options.server, issue_to_apply, patchset_to_apply) print('Downloading patch from %s' % issue_url) try: patchset = obj.get_patch(issue_to_apply, patchset_to_apply) except urllib2.HTTPError as e: print( 'Failed to fetch the patch for issue %d, patchset %d.\n' 'Try visiting %s/%d') % (issue_to_apply, patchset_to_apply, options.server, issue_to_apply) return 1 if options.whitelist: patchset.patches = [ patch for patch in patchset.patches if patch.filename in options.whitelist ] if options.blacklist: patchset.patches = [ patch for patch in patchset.patches if patch.filename not in options.blacklist ] for patch in patchset.patches: print(patch) full_dir = os.path.abspath(options.root_dir) scm_type = scm.determine_scm(full_dir) if scm_type == 'svn': scm_obj = checkout.SvnCheckout(full_dir, None, None, None, None) elif scm_type == 'git': scm_obj = checkout.GitCheckout(full_dir, None, None, None, None) elif scm_type == None: scm_obj = checkout.RawCheckout(full_dir, None, None) else: parser.error('Couldn\'t determine the scm') # TODO(maruel): HACK, remove me. # When run a build slave, make sure buildbot knows that the checkout was # modified. if options.root_dir == 'src' and getpass.getuser() == 'chrome-bot': # See sourcedirIsPatched() in: # http://src.chromium.org/viewvc/chrome/trunk/tools/build/scripts/slave/ # chromium_commands.py?view=markup open('.buildbot-patched', 'w').close() print('\nApplying the patch from %s' % issue_url) try: scm_obj.apply_patch(patchset, verbose=True) except checkout.PatchApplicationFailed as e: print(str(e)) print('CWD=%s' % os.getcwd()) print('Checkout path=%s' % scm_obj.project_path) return 1 if ('DEPS' in map(os.path.basename, patchset.filenames) and not options.ignore_deps): gclient_root = gclient_utils.FindGclientRoot(full_dir) if gclient_root and scm_type: print( 'A DEPS file was updated inside a gclient checkout, running gclient ' 'sync.') gclient_path = os.path.join(BASE_DIR, 'gclient') if sys.platform == 'win32': gclient_path += '.bat' with annotated_gclient.temp_filename(suffix='gclient') as f: cmd = [ gclient_path, 'sync', '--nohooks', '--delete_unversioned_trees', ] if scm_type == 'svn': cmd.extend(['--revision', 'BASE']) if options.revision_mapping: cmd.extend(['--output-json', f]) retcode = subprocess.call(cmd, cwd=gclient_root) if retcode == 0 and options.revision_mapping: revisions = annotated_gclient.parse_got_revision( f, options.revision_mapping) annotated_gclient.emit_buildprops(revisions) return retcode return 0
def main(): # Silence upload.py. rietveld.upload.verbosity = 0 today = datetime.date.today() begin, end = get_previous_quarter(today) default_email = os.environ.get('EMAIL_ADDRESS') if not default_email: user = os.environ.get('USER') if user: default_email = user + '@chromium.org' parser = optparse.OptionParser(description=__doc__) parser.add_option( '--count', action='store_true', help='Just count instead of printing individual issues') parser.add_option( '-r', '--reviewer', metavar='<email>', default=default_email, help='Filter on issue reviewer, default=%default') parser.add_option( '-b', '--begin', metavar='<date>', help='Filter issues created after the date') parser.add_option( '-e', '--end', metavar='<date>', help='Filter issues created before the date') parser.add_option( '-Q', '--last_quarter', action='store_true', help='Use last quarter\'s dates, e.g. %s to %s' % (begin, end)) parser.add_option( '-i', '--instance_url', metavar='<host>', default='http://codereview.chromium.org', help='Host to use, default is %default') auth.add_auth_options(parser) # Remove description formatting parser.format_description = ( lambda _: parser.description) # pylint: disable=no-member options, args = parser.parse_args() auth_config = auth.extract_auth_config_from_options(options) if args: parser.error('Args unsupported') if options.reviewer is None: parser.error('$EMAIL_ADDRESS and $USER are not set, please use -r') print >> sys.stderr, 'Searching for reviews by %s' % options.reviewer if options.last_quarter: options.begin = begin options.end = end print >> sys.stderr, 'Using range %s to %s' % ( options.begin, options.end) else: if options.begin is None or options.end is None: parser.error('Please specify either --last_quarter or --begin and --end') # Validate dates. try: options.begin = dateutil.parser.parse(options.begin).strftime('%Y-%m-%d') options.end = dateutil.parser.parse(options.end).strftime('%Y-%m-%d') except ValueError as e: parser.error('%s: %s - %s' % (e, options.begin, options.end)) if options.count: print_count( options.reviewer, options.begin, options.end, options.instance_url, auth_config) else: print_reviews( options.reviewer, options.begin, options.end, options.instance_url, auth_config) return 0
def main(): # TODO(pgervais,tandrii): split this func, it's still too long. sys.stdout = Unbuffered(sys.stdout) parser = _get_arg_parser() options, args = parser.parse_args() auth_config = auth.extract_auth_config_from_options(options) if options.whitelist and options.blacklist: parser.error('Cannot specify both --whitelist and --blacklist') if options.email and options.email_file: parser.error('-e and -E options are incompatible') if (os.path.isfile(os.path.join(os.getcwd(), 'update.flag')) and not options.force): print 'update.flag file found: bot_update has run and checkout is already ' print 'in a consistent state. No actions will be performed in this step.' return 0 logging.basicConfig( format='%(levelname)5s %(module)11s(%(lineno)4d): %(message)s', level=[logging.WARNING, logging.INFO, logging.DEBUG][ min(2, options.verbose)]) if args: parser.error('Extra argument(s) "%s" not understood' % ' '.join(args)) if not options.issue: parser.error('Require --issue') options.server = options.server.rstrip('/') if not options.server: parser.error('Require a valid server') options.revision_mapping = json.loads(options.revision_mapping) # read email if needed if options.email_file: if not os.path.exists(options.email_file): parser.error('file does not exist: %s' % options.email_file) with open(options.email_file, 'rb') as f: options.email = f.read().strip() print('Connecting to %s' % options.server) # Always try un-authenticated first, except for OAuth2 if options.private_key_file: # OAuth2 authentication rietveld_obj = rietveld.JwtOAuth2Rietveld(options.server, options.email, options.private_key_file) try: properties = rietveld_obj.get_issue_properties(options.issue, False) except urllib2.URLError: logging.exception('failed to fetch issue properties') sys.exit(RETURN_CODE_INFRA_FAILURE) else: # Passing None as auth_config disables authentication. rietveld_obj = rietveld.Rietveld(options.server, None) properties = None # Bad except clauses order (HTTPError is an ancestor class of # ClientLoginError) # pylint: disable=E0701 try: properties = rietveld_obj.get_issue_properties(options.issue, False) except urllib2.HTTPError as e: if e.getcode() != 302: raise if options.no_auth: exit('FAIL: Login detected -- is issue private?') # TODO(maruel): A few 'Invalid username or password.' are printed first, # we should get rid of those. except urllib2.URLError: logging.exception('failed to fetch issue properties') return RETURN_CODE_INFRA_FAILURE except rietveld.upload.ClientLoginError as e: # Fine, we'll do proper authentication. pass if properties is None: rietveld_obj = rietveld.Rietveld(options.server, auth_config, options.email) try: properties = rietveld_obj.get_issue_properties(options.issue, False) except rietveld.upload.ClientLoginError as e: print('Accessing the issue requires proper credentials.') return RETURN_CODE_OTHER_FAILURE except urllib2.URLError: logging.exception('failed to fetch issue properties') return RETURN_CODE_INFRA_FAILURE if not options.patchset: options.patchset = properties['patchsets'][-1] print('No patchset specified. Using patchset %d' % options.patchset) issues_patchsets_to_apply = [(options.issue, options.patchset)] try: depends_on_info = rietveld_obj.get_depends_on_patchset( options.issue, options.patchset) except urllib2.URLError: logging.exception('failed to fetch depends_on_patchset') return RETURN_CODE_INFRA_FAILURE while depends_on_info: depends_on_issue = int(depends_on_info['issue']) depends_on_patchset = int(depends_on_info['patchset']) try: depends_on_info = rietveld_obj.get_depends_on_patchset(depends_on_issue, depends_on_patchset) issues_patchsets_to_apply.insert(0, (depends_on_issue, depends_on_patchset)) except urllib2.HTTPError: print ('The patchset that was marked as a dependency no longer ' 'exists: %s/%d/#ps%d' % ( options.server, depends_on_issue, depends_on_patchset)) print 'Therefore it is likely that this patch will not apply cleanly.' print depends_on_info = None except urllib2.URLError: logging.exception('failed to fetch dependency issue') return RETURN_CODE_INFRA_FAILURE num_issues_patchsets_to_apply = len(issues_patchsets_to_apply) if num_issues_patchsets_to_apply > 1: print print 'apply_issue.py found %d dependent CLs.' % ( num_issues_patchsets_to_apply - 1) print 'They will be applied in the following order:' num = 1 for issue_to_apply, patchset_to_apply in issues_patchsets_to_apply: print ' #%d %s/%d/#ps%d' % ( num, options.server, issue_to_apply, patchset_to_apply) num += 1 print for issue_to_apply, patchset_to_apply in issues_patchsets_to_apply: issue_url = '%s/%d/#ps%d' % (options.server, issue_to_apply, patchset_to_apply) print('Downloading patch from %s' % issue_url) try: patchset = rietveld_obj.get_patch(issue_to_apply, patchset_to_apply) except urllib2.HTTPError: print( 'Failed to fetch the patch for issue %d, patchset %d.\n' 'Try visiting %s/%d') % ( issue_to_apply, patchset_to_apply, options.server, issue_to_apply) # If we got this far, then this is likely missing patchset. # Thus, it's not infra failure. return RETURN_CODE_OTHER_FAILURE except urllib2.URLError: logging.exception( 'Failed to fetch the patch for issue %d, patchset %d', issue_to_apply, patchset_to_apply) return RETURN_CODE_INFRA_FAILURE if options.whitelist: patchset.patches = [patch for patch in patchset.patches if patch.filename in options.whitelist] if options.blacklist: patchset.patches = [patch for patch in patchset.patches if patch.filename not in options.blacklist] for patch in patchset.patches: print(patch) full_dir = os.path.abspath(options.root_dir) scm_type = scm.determine_scm(full_dir) if scm_type == 'svn': scm_obj = checkout.SvnCheckout(full_dir, None, None, None, None) elif scm_type == 'git': scm_obj = checkout.GitCheckout(full_dir, None, None, None, None) elif scm_type == None: scm_obj = checkout.RawCheckout(full_dir, None, None) else: parser.error('Couldn\'t determine the scm') # TODO(maruel): HACK, remove me. # When run a build slave, make sure buildbot knows that the checkout was # modified. if options.root_dir == 'src' and getpass.getuser() == 'chrome-bot': # See sourcedirIsPatched() in: # http://src.chromium.org/viewvc/chrome/trunk/tools/build/scripts/slave/ # chromium_commands.py?view=markup open('.buildbot-patched', 'w').close() print('\nApplying the patch from %s' % issue_url) try: scm_obj.apply_patch(patchset, verbose=True) except checkout.PatchApplicationFailed as e: print(str(e)) print('CWD=%s' % os.getcwd()) print('Checkout path=%s' % scm_obj.project_path) return RETURN_CODE_OTHER_FAILURE if ('DEPS' in map(os.path.basename, patchset.filenames) and not options.ignore_deps): gclient_root = gclient_utils.FindGclientRoot(full_dir) if gclient_root and scm_type: print( 'A DEPS file was updated inside a gclient checkout, running gclient ' 'sync.') gclient_path = os.path.join(BASE_DIR, 'gclient') if sys.platform == 'win32': gclient_path += '.bat' with annotated_gclient.temp_filename(suffix='gclient') as f: cmd = [ gclient_path, 'sync', '--nohooks', '--delete_unversioned_trees', ] if scm_type == 'svn': cmd.extend(['--revision', 'BASE']) if options.revision_mapping: cmd.extend(['--output-json', f]) retcode = subprocess.call(cmd, cwd=gclient_root) if retcode == 0 and options.revision_mapping: revisions = annotated_gclient.parse_got_revision( f, options.revision_mapping) annotated_gclient.emit_buildprops(revisions) return retcode return RETURN_CODE_OK