def new_parse_args(args=None, values=None):
options, args = old_parse_args(args, values)
if not options.issue:
parser.error('Require --issue')
obj = rietveld.Rietveld(options.server, options.user, None)
return options, args, obj
def setUp(self):
self.rietveld = rietveld.Rietveld('url', 'email', 'password')
self.rietveld._send = self._rietveld_send
self.requests = []
def Main(argv):
parser = optparse.OptionParser(usage="%prog [options] <files...>",
version="%prog " + str(__version__))
parser.add_option("-c",
"--commit",
action="store_true",
default=False,
help="Use commit instead of upload checks")
parser.add_option("-u",
"--upload",
action="store_false",
dest='commit',
help="Use upload instead of commit checks")
parser.add_option("-r",
"--recursive",
action="store_true",
help="Act recursively")
parser.add_option("-v",
"--verbose",
action="count",
default=0,
help="Use 2 times for more debug info")
parser.add_option("--name", default='no name')
parser.add_option("--author")
parser.add_option("--description", default='')
parser.add_option("--issue", type='int', default=0)
parser.add_option("--patchset", type='int', default=0)
parser.add_option("--root",
default=os.getcwd(),
help="Search for PRESUBMIT.py up to this directory. "
"If inherit-review-settings-ok is present in this "
"directory, parent directories up to the root file "
"system directories will also be searched.")
parser.add_option("--default_presubmit")
parser.add_option("--may_prompt", action='store_true', default=False)
parser.add_option("--rietveld_url", help=optparse.SUPPRESS_HELP)
parser.add_option("--rietveld_email", help=optparse.SUPPRESS_HELP)
parser.add_option("--rietveld_password", help=optparse.SUPPRESS_HELP)
options, args = parser.parse_args(argv)
if options.verbose >= 2:
logging.basicConfig(level=logging.DEBUG)
elif options.verbose:
logging.basicConfig(level=logging.INFO)
else:
logging.basicConfig(level=logging.ERROR)
change_class, files = load_files(options, args)
if not change_class:
parser.error('For unversioned directory, <files> is not optional.')
logging.info('Found %d file(s).' % len(files))
rietveld_obj = None
if options.rietveld_url:
rietveld_obj = rietveld.Rietveld(options.rietveld_url,
options.rietveld_email,
options.rietveld_password)
try:
results = DoPresubmitChecks(
change_class(options.name, options.description, options.root,
files, options.issue, options.patchset,
options.author), options.commit, options.verbose,
sys.stdout, sys.stdin, options.default_presubmit,
options.may_prompt, rietveld_obj)
return not results.should_continue()
except PresubmitFailure, e:
print >> sys.stderr, e
print >> sys.stderr, 'Maybe your depot_tools is out of date?'
print >> sys.stderr, 'If all fails, contact maruel@'
return 2
def main():
# TODO(pgervais,tandrii): split this func, it's still too long.
sys.stdout = Unbuffered(sys.stdout)
parser = _get_arg_parser()
options, args = parser.parse_args()
auth_config = auth.extract_auth_config_from_options(options)
if options.whitelist and options.blacklist:
parser.error('Cannot specify both --whitelist and --blacklist')
if options.email and options.email_file:
parser.error('-e and -E options are incompatible')
if (os.path.isfile(os.path.join(os.getcwd(), 'update.flag'))
and not options.force):
print 'update.flag file found: bot_update has run and checkout is already '
print 'in a consistent state. No actions will be performed in this step.'
return 0
logging.basicConfig(
format='%(levelname)5s %(module)11s(%(lineno)4d): %(message)s',
level=[logging.WARNING, logging.INFO,
logging.DEBUG][min(2, options.verbose)])
if args:
parser.error('Extra argument(s) "%s" not understood' % ' '.join(args))
if not options.issue:
parser.error('Require --issue')
options.server = options.server.rstrip('/')
if not options.server:
parser.error('Require a valid server')
options.revision_mapping = json.loads(options.revision_mapping)
# read email if needed
if options.email_file:
if not os.path.exists(options.email_file):
parser.error('file does not exist: %s' % options.email_file)
with open(options.email_file, 'rb') as f:
options.email = f.read().strip()
print('Connecting to %s' % options.server)
# Always try un-authenticated first, except for OAuth2
if options.private_key_file:
# OAuth2 authentication
rietveld_obj = rietveld.JwtOAuth2Rietveld(options.server,
options.email,
options.private_key_file)
try:
properties = rietveld_obj.get_issue_properties(
options.issue, False)
except urllib2.URLError:
logging.exception('failed to fetch issue properties')
sys.exit(RETURN_CODE_INFRA_FAILURE)
else:
# Passing None as auth_config disables authentication.
rietveld_obj = rietveld.Rietveld(options.server, None)
properties = None
# Bad except clauses order (HTTPError is an ancestor class of
# ClientLoginError)
# pylint: disable=bad-except-order
try:
properties = rietveld_obj.get_issue_properties(
options.issue, False)
except urllib2.HTTPError as e:
if e.getcode() != 302:
raise
if options.no_auth:
exit('FAIL: Login detected -- is issue private?')
# TODO(maruel): A few 'Invalid username or password.' are printed first,
# we should get rid of those.
except urllib2.URLError:
logging.exception('failed to fetch issue properties')
return RETURN_CODE_INFRA_FAILURE
except rietveld.upload.ClientLoginError as e:
# Fine, we'll do proper authentication.
pass
if properties is None:
rietveld_obj = rietveld.Rietveld(options.server, auth_config,
options.email)
try:
properties = rietveld_obj.get_issue_properties(
options.issue, False)
except rietveld.upload.ClientLoginError as e:
print('Accessing the issue requires proper credentials.')
return RETURN_CODE_OTHER_FAILURE
except urllib2.URLError:
logging.exception('failed to fetch issue properties')
return RETURN_CODE_INFRA_FAILURE
if not options.patchset:
options.patchset = properties['patchsets'][-1]
print('No patchset specified. Using patchset %d' % options.patchset)
issues_patchsets_to_apply = [(options.issue, options.patchset)]
try:
depends_on_info = rietveld_obj.get_depends_on_patchset(
options.issue, options.patchset)
except urllib2.URLError:
logging.exception('failed to fetch depends_on_patchset')
return RETURN_CODE_INFRA_FAILURE
while depends_on_info:
depends_on_issue = int(depends_on_info['issue'])
depends_on_patchset = int(depends_on_info['patchset'])
try:
depends_on_info = rietveld_obj.get_depends_on_patchset(
depends_on_issue, depends_on_patchset)
issues_patchsets_to_apply.insert(
0, (depends_on_issue, depends_on_patchset))
except urllib2.HTTPError:
print(
'The patchset that was marked as a dependency no longer '
'exists: %s/%d/#ps%d' %
(options.server, depends_on_issue, depends_on_patchset))
print 'Therefore it is likely that this patch will not apply cleanly.'
print
depends_on_info = None
except urllib2.URLError:
logging.exception('failed to fetch dependency issue')
return RETURN_CODE_INFRA_FAILURE
num_issues_patchsets_to_apply = len(issues_patchsets_to_apply)
if num_issues_patchsets_to_apply > 1:
print
print 'apply_issue.py found %d dependent CLs.' % (
num_issues_patchsets_to_apply - 1)
print 'They will be applied in the following order:'
num = 1
for issue_to_apply, patchset_to_apply in issues_patchsets_to_apply:
print ' #%d %s/%d/#ps%d' % (num, options.server, issue_to_apply,
patchset_to_apply)
num += 1
print
for issue_to_apply, patchset_to_apply in issues_patchsets_to_apply:
issue_url = '%s/%d/#ps%d' % (options.server, issue_to_apply,
patchset_to_apply)
print('Downloading patch from %s' % issue_url)
try:
patchset = rietveld_obj.get_patch(issue_to_apply,
patchset_to_apply)
except urllib2.HTTPError:
print(
'Failed to fetch the patch for issue %d, patchset %d.\n'
'Try visiting %s/%d') % (issue_to_apply, patchset_to_apply,
options.server, issue_to_apply)
# If we got this far, then this is likely missing patchset.
# Thus, it's not infra failure.
return RETURN_CODE_OTHER_FAILURE
except urllib2.URLError:
logging.exception(
'Failed to fetch the patch for issue %d, patchset %d',
issue_to_apply, patchset_to_apply)
return RETURN_CODE_INFRA_FAILURE
if options.whitelist:
patchset.patches = [
patch for patch in patchset.patches
if patch.filename in options.whitelist
]
if options.blacklist:
patchset.patches = [
patch for patch in patchset.patches
if patch.filename not in options.blacklist
]
for patch in patchset.patches:
print(patch)
if options.extra_patchlevel:
patch.patchlevel += options.extra_patchlevel
full_dir = os.path.abspath(options.root_dir)
scm_type = scm.determine_scm(full_dir)
if scm_type == 'git':
scm_obj = checkout.GitCheckout(full_dir, None, None, None, None)
else:
parser.error('Couldn\'t determine the scm')
print('\nApplying the patch from %s' % issue_url)
try:
scm_obj.apply_patch(patchset, verbose=True)
except checkout.PatchApplicationFailed as e:
print(str(e))
print('CWD=%s' % os.getcwd())
print('Checkout path=%s' % scm_obj.project_path)
return RETURN_CODE_OTHER_FAILURE
if ('DEPS' in map(os.path.basename, patchset.filenames)
and not options.ignore_deps):
gclient_root = gclient_utils.FindGclientRoot(full_dir)
if gclient_root and scm_type:
print(
'A DEPS file was updated inside a gclient checkout, running gclient '
'sync.')
gclient_path = os.path.join(BASE_DIR, 'gclient')
if sys.platform == 'win32':
gclient_path += '.bat'
with annotated_gclient.temp_filename(suffix='gclient') as f:
cmd = [
gclient_path,
'sync',
'--nohooks',
'--delete_unversioned_trees',
]
if options.revision_mapping:
cmd.extend(['--output-json', f])
retcode = subprocess.call(cmd, cwd=gclient_root)
if retcode == 0 and options.revision_mapping:
revisions = annotated_gclient.parse_got_revision(
f, options.revision_mapping)
annotated_gclient.emit_buildprops(revisions)
return retcode
return RETURN_CODE_OK
def main():
# TODO(pgervais): This function is way too long. Split.
sys.stdout = Unbuffered(sys.stdout)
parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
parser.add_option('-v',
'--verbose',
action='count',
default=0,
help='Prints debugging infos')
parser.add_option(
'-e',
'--email',
help='Email address to access rietveld. If not specified, anonymous '
'access will be used.')
parser.add_option(
'-E',
'--email-file',
help='File containing the email address to access rietveld. '
'If not specified, anonymous access will be used.')
parser.add_option(
'-w',
'--password',
help='Password for email addressed. Use - to read password from stdin. '
'if -k is provided, this is the private key file password.')
parser.add_option(
'-k',
'--private-key-file',
help='Path to file containing a private key in p12 format for OAuth2 '
'authentication. Use -w to provide the decrypting password, if any.')
parser.add_option('-i',
'--issue',
type='int',
help='Rietveld issue number')
parser.add_option('-p',
'--patchset',
type='int',
help='Rietveld issue\'s patchset number')
parser.add_option('-r',
'--root_dir',
default=os.getcwd(),
help='Root directory to apply the patch')
parser.add_option('-s',
'--server',
default='http://codereview.chromium.org',
help='Rietveld server')
parser.add_option('--no-auth',
action='store_true',
help='Do not attempt authenticated requests.')
parser.add_option('--revision-mapping',
default='{}',
help='When running gclient, annotate the got_revisions '
'using the revision-mapping.')
parser.add_option('-f',
'--force',
action='store_true',
help='Really run apply_issue, even if .update.flag '
'is detected.')
parser.add_option('-b', '--base_ref', help='DEPRECATED do not use.')
parser.add_option('--whitelist',
action='append',
default=[],
help='Patch only specified file(s).')
parser.add_option('--blacklist',
action='append',
default=[],
help='Don\'t patch specified file(s).')
parser.add_option('-d',
'--ignore_deps',
action='store_true',
help='Don\'t run gclient sync on DEPS changes.')
options, args = parser.parse_args()
if options.whitelist and options.blacklist:
parser.error('Cannot specify both --whitelist and --blacklist')
if options.password and options.private_key_file:
parser.error('-k and -w options are incompatible')
if options.email and options.email_file:
parser.error('-e and -E options are incompatible')
if (os.path.isfile(os.path.join(os.getcwd(), 'update.flag'))
and not options.force):
print 'update.flag file found: bot_update has run and checkout is already '
print 'in a consistent state. No actions will be performed in this step.'
return 0
logging.basicConfig(
format='%(levelname)5s %(module)11s(%(lineno)4d): %(message)s',
level=[logging.WARNING, logging.INFO,
logging.DEBUG][min(2, options.verbose)])
if args:
parser.error('Extra argument(s) "%s" not understood' % ' '.join(args))
if not options.issue:
parser.error('Require --issue')
options.server = options.server.rstrip('/')
if not options.server:
parser.error('Require a valid server')
options.revision_mapping = json.loads(options.revision_mapping)
if options.password == '-':
print('Reading password')
options.password = sys.stdin.readline().strip()
# read email if needed
if options.email_file:
if not os.path.exists(options.email_file):
parser.error('file does not exist: %s' % options.email_file)
with open(options.email_file, 'rb') as f:
options.email = f.read().strip()
print('Connecting to %s' % options.server)
# Always try un-authenticated first, except for OAuth2
if options.private_key_file:
# OAuth2 authentication
obj = rietveld.JwtOAuth2Rietveld(options.server,
options.email,
options.private_key_file,
private_key_password=options.password)
properties = obj.get_issue_properties(options.issue, False)
else:
obj = rietveld.Rietveld(options.server, '', None)
properties = None
# Bad except clauses order (HTTPError is an ancestor class of
# ClientLoginError)
# pylint: disable=E0701
try:
properties = obj.get_issue_properties(options.issue, False)
except urllib2.HTTPError as e:
if e.getcode() != 302:
raise
if options.no_auth:
exit('FAIL: Login detected -- is issue private?')
# TODO(maruel): A few 'Invalid username or password.' are printed first,
# we should get rid of those.
except rietveld.upload.ClientLoginError, e:
# Fine, we'll do proper authentication.
pass
if properties is None:
if options.email is not None:
obj = rietveld.Rietveld(options.server, options.email,
options.password)
try:
properties = obj.get_issue_properties(options.issue, False)
except rietveld.upload.ClientLoginError, e:
if sys.stdout.closed:
print(
'Accessing the issue requires proper credentials.')
return 1
else:
print('Accessing the issue requires login.')
obj = rietveld.Rietveld(options.server, None, None)
try:
properties = obj.get_issue_properties(options.issue, False)
except rietveld.upload.ClientLoginError, e:
print('Accessing the issue requires proper credentials.')
return 1
def main():
# TODO(pgervais): This function is way too long. Split.
sys.stdout = Unbuffered(sys.stdout)
parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
parser.add_option('-v',
'--verbose',
action='count',
default=0,
help='Prints debugging infos')
parser.add_option(
'-e',
'--email',
help='Email address to access rietveld. If not specified, anonymous '
'access will be used.')
parser.add_option(
'-E',
'--email-file',
help='File containing the email address to access rietveld. '
'If not specified, anonymous access will be used.')
parser.add_option(
'-k',
'--private-key-file',
help='Path to file containing a private key in p12 format for OAuth2 '
'authentication with "notasecret" password (as generated by Google '
'Cloud Console).')
parser.add_option('-i',
'--issue',
type='int',
help='Rietveld issue number')
parser.add_option('-p',
'--patchset',
type='int',
help='Rietveld issue\'s patchset number')
parser.add_option('-r',
'--root_dir',
default=os.getcwd(),
help='Root directory to apply the patch')
parser.add_option('-s',
'--server',
default='http://codereview.chromium.org',
help='Rietveld server')
parser.add_option('--no-auth',
action='store_true',
help='Do not attempt authenticated requests.')
parser.add_option('--revision-mapping',
default='{}',
help='When running gclient, annotate the got_revisions '
'using the revision-mapping.')
parser.add_option('-f',
'--force',
action='store_true',
help='Really run apply_issue, even if .update.flag '
'is detected.')
parser.add_option('-b', '--base_ref', help='DEPRECATED do not use.')
parser.add_option('--whitelist',
action='append',
default=[],
help='Patch only specified file(s).')
parser.add_option('--blacklist',
action='append',
default=[],
help='Don\'t patch specified file(s).')
parser.add_option('-d',
'--ignore_deps',
action='store_true',
help='Don\'t run gclient sync on DEPS changes.')
auth.add_auth_options(parser)
options, args = parser.parse_args()
auth_config = auth.extract_auth_config_from_options(options)
if options.whitelist and options.blacklist:
parser.error('Cannot specify both --whitelist and --blacklist')
if options.email and options.email_file:
parser.error('-e and -E options are incompatible')
if (os.path.isfile(os.path.join(os.getcwd(), 'update.flag'))
and not options.force):
print 'update.flag file found: bot_update has run and checkout is already '
print 'in a consistent state. No actions will be performed in this step.'
return 0
logging.basicConfig(
format='%(levelname)5s %(module)11s(%(lineno)4d): %(message)s',
level=[logging.WARNING, logging.INFO,
logging.DEBUG][min(2, options.verbose)])
if args:
parser.error('Extra argument(s) "%s" not understood' % ' '.join(args))
if not options.issue:
parser.error('Require --issue')
options.server = options.server.rstrip('/')
if not options.server:
parser.error('Require a valid server')
options.revision_mapping = json.loads(options.revision_mapping)
# read email if needed
if options.email_file:
if not os.path.exists(options.email_file):
parser.error('file does not exist: %s' % options.email_file)
with open(options.email_file, 'rb') as f:
options.email = f.read().strip()
print('Connecting to %s' % options.server)
# Always try un-authenticated first, except for OAuth2
if options.private_key_file:
# OAuth2 authentication
obj = rietveld.JwtOAuth2Rietveld(options.server, options.email,
options.private_key_file)
properties = obj.get_issue_properties(options.issue, False)
else:
# Passing None as auth_config disables authentication.
obj = rietveld.Rietveld(options.server, None)
properties = None
# Bad except clauses order (HTTPError is an ancestor class of
# ClientLoginError)
# pylint: disable=E0701
try:
properties = obj.get_issue_properties(options.issue, False)
except urllib2.HTTPError as e:
if e.getcode() != 302:
raise
if options.no_auth:
exit('FAIL: Login detected -- is issue private?')
# TODO(maruel): A few 'Invalid username or password.' are printed first,
# we should get rid of those.
except rietveld.upload.ClientLoginError as e:
# Fine, we'll do proper authentication.
pass
if properties is None:
obj = rietveld.Rietveld(options.server, auth_config, options.email)
try:
properties = obj.get_issue_properties(options.issue, False)
except rietveld.upload.ClientLoginError as e:
print('Accessing the issue requires proper credentials.')
return 1
if not options.patchset:
options.patchset = properties['patchsets'][-1]
print('No patchset specified. Using patchset %d' % options.patchset)
issues_patchsets_to_apply = [(options.issue, options.patchset)]
depends_on_info = obj.get_depends_on_patchset(options.issue,
options.patchset)
while depends_on_info:
depends_on_issue = int(depends_on_info['issue'])
depends_on_patchset = int(depends_on_info['patchset'])
try:
depends_on_info = obj.get_depends_on_patchset(
depends_on_issue, depends_on_patchset)
issues_patchsets_to_apply.insert(
0, (depends_on_issue, depends_on_patchset))
except urllib2.HTTPError:
print(
'The patchset that was marked as a dependency no longer '
'exists: %s/%d/#ps%d' %
(options.server, depends_on_issue, depends_on_patchset))
print 'Therefore it is likely that this patch will not apply cleanly.'
print
depends_on_info = None
num_issues_patchsets_to_apply = len(issues_patchsets_to_apply)
if num_issues_patchsets_to_apply > 1:
print
print 'apply_issue.py found %d dependent CLs.' % (
num_issues_patchsets_to_apply - 1)
print 'They will be applied in the following order:'
num = 1
for issue_to_apply, patchset_to_apply in issues_patchsets_to_apply:
print ' #%d %s/%d/#ps%d' % (num, options.server, issue_to_apply,
patchset_to_apply)
num += 1
print
for issue_to_apply, patchset_to_apply in issues_patchsets_to_apply:
issue_url = '%s/%d/#ps%d' % (options.server, issue_to_apply,
patchset_to_apply)
print('Downloading patch from %s' % issue_url)
try:
patchset = obj.get_patch(issue_to_apply, patchset_to_apply)
except urllib2.HTTPError as e:
print(
'Failed to fetch the patch for issue %d, patchset %d.\n'
'Try visiting %s/%d') % (issue_to_apply, patchset_to_apply,
options.server, issue_to_apply)
return 1
if options.whitelist:
patchset.patches = [
patch for patch in patchset.patches
if patch.filename in options.whitelist
]
if options.blacklist:
patchset.patches = [
patch for patch in patchset.patches
if patch.filename not in options.blacklist
]
for patch in patchset.patches:
print(patch)
full_dir = os.path.abspath(options.root_dir)
scm_type = scm.determine_scm(full_dir)
if scm_type == 'svn':
scm_obj = checkout.SvnCheckout(full_dir, None, None, None, None)
elif scm_type == 'git':
scm_obj = checkout.GitCheckout(full_dir, None, None, None, None)
elif scm_type == None:
scm_obj = checkout.RawCheckout(full_dir, None, None)
else:
parser.error('Couldn\'t determine the scm')
# TODO(maruel): HACK, remove me.
# When run a build slave, make sure buildbot knows that the checkout was
# modified.
if options.root_dir == 'src' and getpass.getuser() == 'chrome-bot':
# See sourcedirIsPatched() in:
# http://src.chromium.org/viewvc/chrome/trunk/tools/build/scripts/slave/
# chromium_commands.py?view=markup
open('.buildbot-patched', 'w').close()
print('\nApplying the patch from %s' % issue_url)
try:
scm_obj.apply_patch(patchset, verbose=True)
except checkout.PatchApplicationFailed as e:
print(str(e))
print('CWD=%s' % os.getcwd())
print('Checkout path=%s' % scm_obj.project_path)
return 1
if ('DEPS' in map(os.path.basename, patchset.filenames)
and not options.ignore_deps):
gclient_root = gclient_utils.FindGclientRoot(full_dir)
if gclient_root and scm_type:
print(
'A DEPS file was updated inside a gclient checkout, running gclient '
'sync.')
gclient_path = os.path.join(BASE_DIR, 'gclient')
if sys.platform == 'win32':
gclient_path += '.bat'
with annotated_gclient.temp_filename(suffix='gclient') as f:
cmd = [
gclient_path,
'sync',
'--nohooks',
'--delete_unversioned_trees',
]
if scm_type == 'svn':
cmd.extend(['--revision', 'BASE'])
if options.revision_mapping:
cmd.extend(['--output-json', f])
retcode = subprocess.call(cmd, cwd=gclient_root)
if retcode == 0 and options.revision_mapping:
revisions = annotated_gclient.parse_got_revision(
f, options.revision_mapping)
annotated_gclient.emit_buildprops(revisions)
return retcode
return 0
def main():
parser = optparse.OptionParser(
description=sys.modules['__main__'].__doc__)
project_choices = projects.supported_projects()
parser.add_option('-v', '--verbose', action='store_true')
parser.add_option(
'--no-dry-run',
action='store_false',
dest='dry_run',
default=True,
help='Run for real instead of dry-run mode which is the default. '
'WARNING: while the CQ won\'t touch rietveld in dry-run mode, the '
'Try Server will. So it is recommended to use --only-issue')
parser.add_option(
'--only-issue',
type='int',
help='Limits to a single issue. Useful for live testing; WARNING: it '
'will fake that the issue has the CQ bit set, so only try with an '
'issue you don\'t mind about.')
parser.add_option(
'--fake',
action='store_true',
help='Run with a fake checkout to speed up testing')
parser.add_option(
'--no-try',
action='store_true',
help='Don\'t send try jobs.')
parser.add_option(
'-p',
'--poll-interval',
type='int',
default=10,
help='Minimum delay between each polling loop, default: %default')
parser.add_option(
'--query-only',
action='store_true',
help='Return internal state')
parser.add_option(
'--project',
choices=project_choices,
help='Project to run the commit queue against: %s' %
', '.join(project_choices))
parser.add_option(
'-u',
'--user',
default='*****@*****.**',
help='User to use instead of %default')
options, args = parser.parse_args()
if args:
parser.error('Unsupported args: %s' % args)
if not options.project:
parser.error('Need to pass a valid project to --project.\nOptions are: %s' %
', '.join(project_choices))
logging.getLogger().setLevel(logging.DEBUG)
if options.verbose:
level = logging.DEBUG
else:
level = logging.INFO
console_logging = logging.StreamHandler()
console_logging.setFormatter(logging.Formatter(
'%(asctime)s %(levelname)7s %(message)s'))
console_logging.setLevel(level)
logging.getLogger().addHandler(console_logging)
log_directory = 'logs-' + options.project
if not os.path.exists(log_directory):
os.mkdir(log_directory)
logging_rotating_file = logging.handlers.RotatingFileHandler(
filename=os.path.join(log_directory, 'commit_queue.log'),
maxBytes= 10*1024*1024,
backupCount=50)
logging_rotating_file.setLevel(logging.DEBUG)
logging_rotating_file.setFormatter(logging.Formatter(
'%(asctime)s %(levelname)-8s %(module)15s(%(lineno)4d): %(message)s'))
logging.getLogger().addHandler(logging_rotating_file)
try:
work_dir = os.path.join(ROOT_DIR, 'workdir')
# Use our specific subversion config.
checkout.SvnMixIn.svn_config = checkout.SvnConfig(
os.path.join(ROOT_DIR, 'subversion_config'))
url = 'https://chromiumcodereview.appspot.com'
gaia_creds = creds.Credentials(os.path.join(work_dir, '.gaia_pwd'))
if options.dry_run:
logging.debug('Dry run - skipping SCM check.')
if options.only_issue:
print(
'Using read-only Rietveld; using only issue %d' %
options.only_issue)
else:
print('Using read-only Rietveld')
# Make sure rietveld is not modified.
rietveld_obj = ReadOnlyRietveld(
url,
options.user,
gaia_creds.get(options.user),
None,
options.only_issue)
else:
AlertOnUncleanCheckout()
print('WARNING: The Commit Queue is going to commit stuff')
if options.only_issue:
print('Using only issue %d' % options.only_issue)
rietveld_obj = OnlyIssueRietveld(
url,
options.user,
gaia_creds.get(options.user),
None,
options.only_issue)
else:
rietveld_obj = rietveld.Rietveld(
url,
options.user,
gaia_creds.get(options.user),
None)
pc = projects.load_project(
options.project,
options.user,
work_dir,
rietveld_obj,
options.no_try)
if options.dry_run:
if options.fake:
# Disable the checkout.
print 'Using no checkout'
pc.context.checkout = FakeCheckout()
else:
print 'Using read-only checkout'
pc.context.checkout = checkout.ReadOnlyCheckout(pc.context.checkout)
# Save pushed events on disk.
print 'Using read-only chromium-status interface'
pc.context.status = async_push.AsyncPushStore()
db_path = os.path.join(work_dir, pc.context.checkout.project_name + '.json')
if os.path.isfile(db_path):
try:
pc.load(db_path)
except ValueError:
os.remove(db_path)
sig_handler.installHandlers(
signal.SIGINT,
signal.SIGHUP
)
# Sync every 5 minutes.
SYNC_DELAY = 5*60
try:
if options.query_only:
pc.look_for_new_pending_commit()
pc.update_status()
print(str(pc.queue))
return 0
now = time.time()
next_loop = now + options.poll_interval
# First sync is on second loop.
next_sync = now + options.poll_interval * 2
while True:
# In theory, we would gain in performance to parallelize these tasks. In
# practice I'm not sure it matters.
pc.look_for_new_pending_commit()
pc.process_new_pending_commit()
pc.update_status()
pc.scan_results()
if sig_handler.getTriggeredSignals():
raise KeyboardInterrupt()
# Save the db at each loop. The db can easily be in the 1mb range so
# it's slowing down the CQ a tad but it in the 100ms range even for that
# size.
pc.save(db_path)
# More than a second to wait and due to sync.
now = time.time()
if (next_loop - now) >= 1 and (next_sync - now) <= 0:
if sys.stdout.isatty():
sys.stdout.write('Syncing while waiting \r')
sys.stdout.flush()
try:
pc.context.checkout.prepare(None)
except subprocess2.CalledProcessError as e:
# Don't crash, most of the time it's the svn server that is dead.
# How fun. Send a stack trace to annoy the maintainer.
errors.send_stack(e)
next_sync = time.time() + SYNC_DELAY
now = time.time()
next_loop = max(now, next_loop)
while True:
# Abort if any signals are set
if sig_handler.getTriggeredSignals():
raise KeyboardInterrupt()
delay = next_loop - now
if delay <= 0:
break
if sys.stdout.isatty():
sys.stdout.write('Sleeping for %1.1f seconds \r' % delay)
sys.stdout.flush()
time.sleep(min(delay, 0.1))
now = time.time()
if sys.stdout.isatty():
sys.stdout.write('Running (please do not interrupt) \r')
sys.stdout.flush()
next_loop = time.time() + options.poll_interval
finally:
print >> sys.stderr, 'Saving db... '
pc.save(db_path)
pc.close()
print >> sys.stderr, 'Done! '
except KeyboardInterrupt as e:
print 'Bye bye'
# 23 is an arbitrary value to signal loop.sh that it must stop looping.
return 23
except SystemExit as e:
traceback.print_exc()
print >> sys.stderr, ('Tried to exit: %s', e)
return e.code
except errors.ConfigurationError as e:
parser.error(str(e))
return 1
return 0
def main():
sys.stdout = Unbuffered(sys.stdout)
parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
parser.add_option(
'-v', '--verbose', action='count', default=0,
help='Prints debugging infos')
parser.add_option(
'-e', '--email',
help='Email address to access rietveld. If not specified, anonymous '
'access will be used.')
parser.add_option(
'-w', '--password', default=None,
help='Password for email addressed. Use - to read password from stdin.')
parser.add_option(
'-i', '--issue', type='int', help='Rietveld issue number')
parser.add_option(
'-p', '--patchset', type='int', help='Rietveld issue\'s patchset number')
parser.add_option(
'-r',
'--root_dir',
default=os.getcwd(),
help='Root directory to apply the patch')
parser.add_option(
'-s',
'--server',
default='http://codereview.chromium.org',
help='Rietveld server')
parser.add_option('--no-auth', action='store_true',
help='Do not attempt authenticated requests.')
options, args = parser.parse_args()
logging.basicConfig(
format='%(levelname)5s %(module)11s(%(lineno)4d): %(message)s',
level=[logging.WARNING, logging.INFO, logging.DEBUG][
min(2, options.verbose)])
if args:
parser.error('Extra argument(s) "%s" not understood' % ' '.join(args))
if not options.issue:
parser.error('Require --issue')
options.server = options.server.rstrip('/')
if not options.server:
parser.error('Require a valid server')
if options.password == '-':
print('Reading password')
options.password = sys.stdin.readline().strip()
print('Connecting to %s' % options.server)
# Always try un-authenticated first.
# TODO(maruel): Use OAuth2 properly so we don't hit rate-limiting on login
# attempts.
# Bad except clauses order (HTTPError is an ancestor class of
# ClientLoginError)
# pylint: disable=E0701
obj = rietveld.Rietveld(options.server, '', None)
properties = None
try:
properties = obj.get_issue_properties(options.issue, False)
except urllib2.HTTPError, e:
if e.getcode() != 302:
raise
elif options.no_auth:
exit('FAIL: Login detected -- is issue private?')
properties = None
try:
properties = obj.get_issue_properties(options.issue, False)
except urllib2.HTTPError, e:
if e.getcode() != 302:
raise
elif options.no_auth:
exit('FAIL: Login detected -- is issue private?')
# TODO(maruel): A few 'Invalid username or password.' are printed first, we
# should get rid of those.
except rietveld.upload.ClientLoginError, e:
# Fine, we'll do proper authentication.
pass
if properties is None:
if options.email is not None:
obj = rietveld.Rietveld(options.server, options.email, options.password)
try:
properties = obj.get_issue_properties(options.issue, False)
except rietveld.upload.ClientLoginError, e:
if sys.stdout.closed:
print('Accessing the issue requires proper credentials.')
return 1
else:
print('Accessing the issue requires login.')
obj = rietveld.Rietveld(options.server, None, None)
try:
properties = obj.get_issue_properties(options.issue, False)
except rietveld.upload.ClientLoginError, e:
print('Accessing the issue requires proper credentials.')
return 1
