def test_all(self):
# Make sure it's possible to load each project.
mapping = {
'chromium': {
'lines': [
'root_dir/.chromium_status_pwd',
'chromium-status password',
['foo'],
],
'pre_patch_verifiers': ['project_bases', 'reviewer_lgtm'],
'verifiers': ['presubmit', 'tree status'],
},
'chromium_deps': {
'lines': [
'root_dir/.chromium_status_pwd',
'chromium-status password',
['foo'],
],
'pre_patch_verifiers': ['project_bases', 'reviewer_lgtm'],
'verifiers': ['presubmit'],
},
'gyp': {
'lines': [
'root_dir/.chromium_status_pwd',
'chromium-status password',
['foo'],
],
'pre_patch_verifiers': ['project_bases', 'reviewer_lgtm'],
'verifiers': ['tree status'],
},
'nacl': {
'lines': [
'root_dir/.chromium_status_pwd',
'chromium-status password',
['foo'],
],
'pre_patch_verifiers': ['project_bases', 'reviewer_lgtm'],
'verifiers': ['tree status'],
},
'tools': {
'lines': [
'root_dir/.chromium_status_pwd',
'chromium-status password',
['foo'],
],
'pre_patch_verifiers': ['project_bases', 'reviewer_lgtm'],
'verifiers': ['presubmit'],
},
}
for project in sorted(projects.supported_projects()):
logging.debug(project)
self.assertEquals([], self.read_lines)
expected = mapping.pop(project)
self.read_lines = [expected['lines']]
p = projects.load_project(project, 'user', 'root_dir',
self.context.rietveld, True)
self.assertEquals(expected['pre_patch_verifiers'],
[x.name for x in p.pre_patch_verifiers],
(expected['pre_patch_verifiers'],
[x.name
for x in p.pre_patch_verifiers], project))
self.assertEquals(
expected['verifiers'], [x.name for x in p.verifiers],
(expected['verifiers'], [x.name
for x in p.verifiers], project))
if project == 'tools':
self.maxDiff = None
# Add special checks for it.
project_bases_verifier = p.pre_patch_verifiers[0]
self.assertEquals([
'^svn\\:\\/\\/svn\\.chromium\\.org\\/chrome/trunk/tools(|/.*)$',
'^svn\\:\\/\\/chrome\\-svn\\/chrome/trunk/tools(|/.*)$',
'^svn\\:\\/\\/chrome\\-svn\\.corp\\/chrome/trunk/tools(|/.*)$',
'^svn\\:\\/\\/chrome\\-svn\\.corp\\.google\\.com\\/chrome/trunk/'
'tools(|/.*)$',
'^http\\:\\/\\/src\\.chromium\\.org\\/svn/trunk/tools(|/.*)$',
'^https\\:\\/\\/src\\.chromium\\.org\\/svn/trunk/tools(|/.*)$',
'^http\\:\\/\\/src\\.chromium\\.org\\/chrome/trunk/tools(|/.*)$',
'^https\\:\\/\\/src\\.chromium\\.org\\/chrome/trunk/tools(|/.*)$',
'^https\\:\\/\\/git\\.chromium\\.org\\/chromium\\/tools\\/'
'([a-z\\-_]+)\\.git\\@[a-z\\-_]+$',
'^http\\:\\/\\/git\\.chromium\\.org\\/chromium\\/tools\\/'
'([a-z\\-_]+)\\.git\\@[a-z\\-_]+$',
'^https\\:\\/\\/git\\.chromium\\.org\\/git\\/chromium\\/tools\\/'
'([a-z\\-_]+)\\@[a-z\\-_]+$',
'^http\\:\\/\\/git\\.chromium\\.org\\/git\\/chromium\\/tools\\/'
'([a-z\\-_]+)\\@[a-z\\-_]+$',
'^https\\:\\/\\/git\\.chromium\\.org\\/git\\/chromium\\/tools\\/'
'([a-z\\-_]+)\\.git\\@[a-z\\-_]+$',
'^http\\:\\/\\/git\\.chromium\\.org\\/git\\/chromium\\/tools\\/'
'([a-z\\-_]+)\\.git\\@[a-z\\-_]+$',
], project_bases_verifier.project_bases)
self.assertEquals({}, mapping)
def test_all(self):
# Make sure it's possible to load each project.
self.time = [1] * 2
root_dir = os.path.join(os.getcwd(), 'root_dir')
chromium_status_pwd = os.path.join(root_dir, '.chromium_status_pwd')
skia_status_pwd = os.path.join(root_dir, '.skia_status_pwd')
mapping = {
'blink': {
'lines': [
chromium_status_pwd,
'chromium-status password',
['foo'],
],
'pre_patch_verifiers': ['project_bases', 'reviewer_lgtm'],
'verifiers': ['try job rietveld', 'tree status'],
},
'chromium': {
'lines': [
chromium_status_pwd,
'chromium-status password',
['foo'],
],
'pre_patch_verifiers': ['project_bases', 'reviewer_lgtm'],
'verifiers': ['try job rietveld', 'tree status'],
},
'chromium_deps': {
'lines': [
chromium_status_pwd,
'chromium-status password',
['foo'],
],
'pre_patch_verifiers': ['project_bases', 'reviewer_lgtm'],
'verifiers': ['presubmit'],
},
'gyp': {
'lines': [
chromium_status_pwd,
'chromium-status password',
['foo'],
],
'pre_patch_verifiers': ['project_bases', 'reviewer_lgtm'],
'verifiers': ['tree status'],
},
'nacl': {
'lines': [
chromium_status_pwd,
'chromium-status password',
['foo'],
],
'pre_patch_verifiers': ['project_bases', 'reviewer_lgtm'],
'verifiers': ['presubmit', 'tree status'],
},
'skia': {
'lines': [
skia_status_pwd,
'skia-status password',
['foo'],
],
'pre_patch_verifiers': ['project_bases', 'reviewer_lgtm'],
'verifiers': ['presubmit', 'tree status'],
},
'tools': {
'lines': [
chromium_status_pwd,
'chromium-status password',
['foo'],
],
'pre_patch_verifiers': ['project_bases', 'reviewer_lgtm'],
'verifiers': ['presubmit'],
},
}
for project in sorted(projects.supported_projects()):
logging.debug(project)
self.assertEqual([], self.read_lines)
expected = mapping.pop(project)
self.read_lines = [expected['lines']]
p = projects.load_project(project, 'user', root_dir,
self.context.rietveld, True)
self.assertEqual(expected['pre_patch_verifiers'],
[x.name for x in p.pre_patch_verifiers],
(expected['pre_patch_verifiers'],
[x.name
for x in p.pre_patch_verifiers], project))
self.assertEqual(
expected['verifiers'], [x.name for x in p.verifiers],
(expected['verifiers'], [x.name
for x in p.verifiers], project))
if project == 'tools':
# Add special checks for it.
project_bases_verifier = p.pre_patch_verifiers[0]
branch = '\\@[a-zA-Z0-9\\-_\\.]+$'
self.assertEqual(
[
# svn
'^svn\\:\\/\\/svn\\.chromium\\.org\\/chrome/trunk/tools(|/.*)$',
'^svn\\:\\/\\/chrome\\-svn\\/chrome/trunk/tools(|/.*)$',
'^svn\\:\\/\\/chrome\\-svn\\.corp\\/chrome/trunk/tools(|/.*)$',
'^svn\\:\\/\\/chrome\\-svn\\.corp\\.google\\.com\\/chrome/trunk/'
'tools(|/.*)$',
'^http\\:\\/\\/src\\.chromium\\.org\\/svn/trunk/tools(|/.*)$',
'^https\\:\\/\\/src\\.chromium\\.org\\/svn/trunk/tools(|/.*)$',
'^http\\:\\/\\/src\\.chromium\\.org\\/chrome/trunk/tools(|/.*)$',
'^https\\:\\/\\/src\\.chromium\\.org\\/chrome/trunk/tools(|/.*)$',
# git
'^https?\\:\\/\\/git\\.chromium\\.org\\/git\\/chromium\\/tools\\/'
'([a-z0-9\\-_]+)(?:\\.git)?' + branch,
'^https?\\:\\/\\/git\\.chromium\\.org\\/chromium\\/tools\\/'
'([a-z0-9\\-_]+)(?:\\.git)?' + branch,
'^https?\\:\\/\\/chromium\\.googlesource\\.com\\/chromium\\/tools'
'\\/([a-z0-9\\-_]+)(?:\\.git)?' + branch,
],
project_bases_verifier.project_bases)
self.assertEqual({}, mapping)
def test_loaded(self):
members = ('chromium', 'chromium_deps', 'gyp', 'nacl', 'tools')
self.assertEquals(sorted(members),
sorted(projects.supported_projects()))
def main():
parser = optparse.OptionParser(
description=sys.modules['__main__'].__doc__)
project_choices = projects.supported_projects()
parser.add_option('-v', '--verbose', action='store_true')
parser.add_option(
'--no-dry-run',
action='store_false',
dest='dry_run',
default=True,
help='Run for real instead of dry-run mode which is the default. '
'WARNING: while the CQ won\'t touch rietveld in dry-run mode, the '
'Try Server will. So it is recommended to use --only-issue')
parser.add_option(
'--only-issue',
type='int',
help='Limits to a single issue. Useful for live testing; WARNING: it '
'will fake that the issue has the CQ bit set, so only try with an '
'issue you don\'t mind about.')
parser.add_option(
'--fake',
action='store_true',
help='Run with a fake checkout to speed up testing')
parser.add_option(
'--no-try',
action='store_true',
help='Don\'t send try jobs.')
parser.add_option(
'-p',
'--poll-interval',
type='int',
default=10,
help='Minimum delay between each polling loop, default: %default')
parser.add_option(
'--query-only',
action='store_true',
help='Return internal state')
parser.add_option(
'--project',
choices=project_choices,
help='Project to run the commit queue against: %s' %
', '.join(project_choices))
parser.add_option(
'-u',
'--user',
default='*****@*****.**',
help='User to use instead of %default')
options, args = parser.parse_args()
if args:
parser.error('Unsupported args: %s' % args)
if not options.project:
parser.error('Need to pass a valid project to --project.\nOptions are: %s' %
', '.join(project_choices))
logging.getLogger().setLevel(logging.DEBUG)
if options.verbose:
level = logging.DEBUG
else:
level = logging.INFO
console_logging = logging.StreamHandler()
console_logging.setFormatter(logging.Formatter(
'%(asctime)s %(levelname)7s %(message)s'))
console_logging.setLevel(level)
logging.getLogger().addHandler(console_logging)
log_directory = 'logs-' + options.project
if not os.path.exists(log_directory):
os.mkdir(log_directory)
logging_rotating_file = logging.handlers.RotatingFileHandler(
filename=os.path.join(log_directory, 'commit_queue.log'),
maxBytes= 10*1024*1024,
backupCount=50)
logging_rotating_file.setLevel(logging.DEBUG)
logging_rotating_file.setFormatter(logging.Formatter(
'%(asctime)s %(levelname)-8s %(module)15s(%(lineno)4d): %(message)s'))
logging.getLogger().addHandler(logging_rotating_file)
try:
work_dir = os.path.join(ROOT_DIR, 'workdir')
# Use our specific subversion config.
checkout.SvnMixIn.svn_config = checkout.SvnConfig(
os.path.join(ROOT_DIR, 'subversion_config'))
url = 'https://chromiumcodereview.appspot.com'
gaia_creds = creds.Credentials(os.path.join(work_dir, '.gaia_pwd'))
if options.dry_run:
logging.debug('Dry run - skipping SCM check.')
if options.only_issue:
print(
'Using read-only Rietveld; using only issue %d' %
options.only_issue)
else:
print('Using read-only Rietveld')
# Make sure rietveld is not modified.
rietveld_obj = ReadOnlyRietveld(
url,
options.user,
gaia_creds.get(options.user),
None,
options.only_issue)
else:
AlertOnUncleanCheckout()
print('WARNING: The Commit Queue is going to commit stuff')
if options.only_issue:
print('Using only issue %d' % options.only_issue)
rietveld_obj = OnlyIssueRietveld(
url,
options.user,
gaia_creds.get(options.user),
None,
options.only_issue)
else:
rietveld_obj = rietveld.Rietveld(
url,
options.user,
gaia_creds.get(options.user),
None)
pc = projects.load_project(
options.project,
options.user,
work_dir,
rietveld_obj,
options.no_try)
if options.dry_run:
if options.fake:
# Disable the checkout.
print 'Using no checkout'
pc.context.checkout = FakeCheckout()
else:
print 'Using read-only checkout'
pc.context.checkout = checkout.ReadOnlyCheckout(pc.context.checkout)
# Save pushed events on disk.
print 'Using read-only chromium-status interface'
pc.context.status = async_push.AsyncPushStore()
db_path = os.path.join(work_dir, pc.context.checkout.project_name + '.json')
if os.path.isfile(db_path):
try:
pc.load(db_path)
except ValueError:
os.remove(db_path)
sig_handler.installHandlers(
signal.SIGINT,
signal.SIGHUP
)
# Sync every 5 minutes.
SYNC_DELAY = 5*60
try:
if options.query_only:
pc.look_for_new_pending_commit()
pc.update_status()
print(str(pc.queue))
return 0
now = time.time()
next_loop = now + options.poll_interval
# First sync is on second loop.
next_sync = now + options.poll_interval * 2
while True:
# In theory, we would gain in performance to parallelize these tasks. In
# practice I'm not sure it matters.
pc.look_for_new_pending_commit()
pc.process_new_pending_commit()
pc.update_status()
pc.scan_results()
if sig_handler.getTriggeredSignals():
raise KeyboardInterrupt()
# Save the db at each loop. The db can easily be in the 1mb range so
# it's slowing down the CQ a tad but it in the 100ms range even for that
# size.
pc.save(db_path)
# More than a second to wait and due to sync.
now = time.time()
if (next_loop - now) >= 1 and (next_sync - now) <= 0:
if sys.stdout.isatty():
sys.stdout.write('Syncing while waiting \r')
sys.stdout.flush()
try:
pc.context.checkout.prepare(None)
except subprocess2.CalledProcessError as e:
# Don't crash, most of the time it's the svn server that is dead.
# How fun. Send a stack trace to annoy the maintainer.
errors.send_stack(e)
next_sync = time.time() + SYNC_DELAY
now = time.time()
next_loop = max(now, next_loop)
while True:
# Abort if any signals are set
if sig_handler.getTriggeredSignals():
raise KeyboardInterrupt()
delay = next_loop - now
if delay <= 0:
break
if sys.stdout.isatty():
sys.stdout.write('Sleeping for %1.1f seconds \r' % delay)
sys.stdout.flush()
time.sleep(min(delay, 0.1))
now = time.time()
if sys.stdout.isatty():
sys.stdout.write('Running (please do not interrupt) \r')
sys.stdout.flush()
next_loop = time.time() + options.poll_interval
finally:
print >> sys.stderr, 'Saving db... '
pc.save(db_path)
pc.close()
print >> sys.stderr, 'Done! '
except KeyboardInterrupt as e:
print 'Bye bye'
# 23 is an arbitrary value to signal loop.sh that it must stop looping.
return 23
except SystemExit as e:
traceback.print_exc()
print >> sys.stderr, ('Tried to exit: %s', e)
return e.code
except errors.ConfigurationError as e:
parser.error(str(e))
return 1
return 0