def main(argv):
start_time = time.time()
parser = optparse.OptionParser()
parser.add_option('-v', '--verbose', action='store_true')
options, args = parser.parse_args(argv[1:])
if args:
parser.error('Unknown arguments: %s' % args)
num_skipped = len(BLACKLIST)
masters_list = GetMasterCmds(
masters=master_cfg_utils.GetMasters(include_internal=False),
blacklist=BLACKLIST,
pythonpaths=None)
build_internal = os.path.join(BASE_DIR, '..', 'build_internal')
if os.path.exists(build_internal):
internal_test_data = chromium_utils.ParsePythonCfg(
os.path.join(build_internal, 'tests', 'internal_masters_cfg.py'),
fail_hard=True)
internal_cfg = internal_test_data['masters_cfg_test']
num_skipped += len(internal_cfg['blacklist'])
masters_list.extend(GetMasterCmds(
masters=master_cfg_utils.GetMasters(include_public=False),
blacklist=internal_cfg['blacklist'],
pythonpaths=[os.path.join(build_internal, p)
for p in internal_cfg['paths']]))
with master_cfg_utils.TemporaryMasterPasswords():
processes = [subprocess.Popen([
sys.executable, os.path.join(BASE_DIR, 'scripts', 'slave', 'runbuild.py'),
cmd.name, '--test-config'], stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, env=cmd.env) for cmd in masters_list]
results = [(proc.communicate()[0], proc.returncode) for proc in processes]
def GetCommandStr(cmd, cmd_output):
out = [cmd.path]
out.extend('> ' + line for line in cmd_output.splitlines())
return '\n'.join(out + [''])
if options.verbose:
for cmd, (out, code) in zip(masters_list, results):
# Failures will be printed below
if code == 0 and out:
print GetCommandStr(cmd, out)
failures = [(cmd, out) for cmd, (out, r) in zip(masters_list, results) if r]
if failures:
print '\nFAILURE The following master.cfg files did not load:\n'
for cmd, out in failures:
print GetCommandStr(cmd, out)
test_time = round(time.time() - start_time, 1)
print 'Parsed %d masters successfully, %d failed, %d skipped in %gs.' % (
len(masters_list), len(failures), num_skipped, test_time)
return bool(failures)
def get_factory_properties_from_disk(mastername, buildername):
master_list = master_cfg_utils.GetMasters()
master_path = None
for name, path in master_list:
if name == mastername:
master_path = path
if not master_path:
raise LookupError('master "%s" not found.' % mastername)
script_path = os.path.join(BUILD_ROOT, 'scripts', 'tools',
'dump_master_cfg.py')
with namedTempFile() as fname:
dump_cmd = [sys.executable, script_path, master_path, fname]
proc = subprocess.Popen(dump_cmd,
cwd=BUILD_ROOT,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = proc.communicate()
exit_code = proc.returncode
if exit_code:
raise LookupError(
'Failed to get the master config; dump_master_cfg %s'
'returned %d):\n%s\n%s\n' % (mastername, exit_code, out, err))
with open(fname, 'rU') as f:
config = json.load(f)
# Now extract just the factory properties for the requested builder
# from the master config.
props = {}
found = False
for builder_dict in config['builders']:
if builder_dict['name'] == buildername:
found = True
factory_properties = builder_dict['factory']['properties']
for name, (value, _) in factory_properties.items():
props[name] = value
if not found:
raise LookupError('builder "%s" not found on in master "%s"' %
(buildername, mastername))
if 'recipe' not in props:
raise LookupError('Cannot find recipe for %s on %s' %
(buildername, mastername))
return props
def get_factory_properties_from_disk(workdir, mastername, buildername):
master_list = master_cfg_utils.GetMasters()
master_path = None
for name, path in master_list:
if name == mastername:
master_path = path
if not master_path:
raise LookupError('master "%s" not found.' % mastername)
script_path = os.path.join(env.Build, 'scripts', 'tools',
'dump_master_cfg.py')
master_json = os.path.join(workdir, 'dump_master_cfg.json')
dump_cmd = [sys.executable, script_path, master_path, master_json]
proc = subprocess.Popen(dump_cmd,
cwd=env.Build,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = proc.communicate()
if proc.returncode:
raise LookupError('Failed to get the master config; running %r in %r '
'returned exit code %d\nstdout: %s\nstderr: %s' %
(dump_cmd, env.Build, proc.returncode, out, err))
with open(master_json, 'rU') as f:
config = json.load(f)
# Now extract just the factory properties for the requested builder
# from the master config.
props = {}
found = False
for builder_dict in config['builders']:
if builder_dict['name'] == buildername:
found = True
factory_properties = builder_dict['factory']['properties']
for name, (value, _) in factory_properties.items():
props[name] = value
if not found:
raise LookupError('builder "%s" not found on in master "%s"' %
(buildername, mastername))
if 'recipe' not in props:
raise LookupError('Cannot find recipe for %s on %s' %
(buildername, mastername))
return props
def execute(options):
if options.list_masters:
masterpairs = master_cfg_utils.GetMasters()
master_cfg_utils.PrettyPrintMasters(masterpairs)
return 0
if options.master_dir:
config = master_cfg_utils.LoadConfig(options.master_dir,
options.master_cfg)
else:
path = master_cfg_utils.ChooseMaster(options.mastername)
if not path:
return 2
config = master_cfg_utils.LoadConfig(path,
config_file=options.master_cfg)
if not config:
return 2
mastername = config['BuildmasterConfig']['properties']['mastername']
builders = config['BuildmasterConfig']['builders']
options.build_properties.update(config['BuildmasterConfig'].get(
'properties', {}))
if options.list_builders:
master_cfg_utils.PrettyPrintBuilders(builders, mastername)
return 0
if options.test_config:
for builder in builders:
# We need to provide a slavename, so just pick the first one
# the builder has.
builder['slavename'] = builder['slavenames'][0]
execute_builder(builder, mastername, options)
return 0
my_builder = master_cfg_utils.ChooseBuilder(builders, options.spec)
return execute_builder(my_builder, mastername, options)
def main(options):
if options.list_masters:
masterpairs = master_cfg_utils.GetMasters()
master_cfg_utils.PrettyPrintMasters(masterpairs)
return 0
if options.master_dir:
path = options.master_dir
elif options.mastername:
path = master_cfg_utils.ChooseMaster(options.mastername)
if not path:
return 2
else:
path = options.filename
if os.path.isdir(path):
print path
for directory in locateBuilderDirs(path, options):
if not processDirectory(directory, options):
if not options.commit:
print
print '*** errors encountered but commit was not set '
return 2
if not options.commit:
print
print '*** re-run with --commit to overwrite changes ***'
else:
builder = loadBuilder(options.builderpath)
if not processBuild(path, builder, options):
if not options.commit:
print
print '*** errors encountered but commit was not set '
return 2
if not options.commit:
print
print '*** re-run with --commit to overwrite changes ***'
return 0