def __init__(self,
write_bucket,
read_buckets,
gsutil=None,
call=subprocess.call,
download=HttpDownload):
"""Init for this class.
Args:
write_bucket: Google storage location to write to.
read_buckets: Google storage locations to read from in preferred order.
gsutil: List of cmd components needed to call gsutil.
call: Testing hook to intercept command invocation.
download: Testing hook to intercept download.
"""
if gsutil is None:
try:
# Require that gsutil be Python if it is specified in the environment.
gsutil = [
sys.executable,
file_tools.Which(os.environ.get('GSUTIL', 'gsutil'),
require_executable=False)
]
except file_tools.ExecutableNotFound:
gsutil = ['gsutil']
assert isinstance(gsutil, list)
assert isinstance(read_buckets, list)
self._gsutil = gsutil
self._write_bucket = write_bucket
self._read_buckets = read_buckets
self._call = call
self._download = download
def runcmd(subst, command, stdout, **kwargs):
check_call_kwargs = kwargs.copy()
command = command[:]
cwd = subst.SubstituteAbsPaths(check_call_kwargs.get('cwd', '.'))
subst.SetCwd(cwd)
check_call_kwargs['cwd'] = cwd
# Extract paths from kwargs and add to the command environment.
path_dirs = []
if 'path_dirs' in check_call_kwargs:
path_dirs = [
subst.Substitute(dirname)
for dirname in check_call_kwargs['path_dirs']
]
del check_call_kwargs['path_dirs']
check_call_kwargs['env'] = PlatformEnvironment(path_dirs)
if isinstance(command, str):
command = subst.Substitute(command)
else:
command = [subst.Substitute(arg) for arg in command]
paths = check_call_kwargs['env']['PATH'].split(os.pathsep)
command[0] = file_tools.Which(command[0], paths=paths)
if stdout is not None:
stdout = subst.SubstituteAbsPaths(stdout)
log_tools.CheckCall(command, stdout=stdout, **check_call_kwargs)
def SystemSummary(self):
"""Gather a string describing intrinsic properties of the current machine.
Ideally this would capture anything relevant about the current machine that
would cause build output to vary (other than build recipe + inputs).
"""
if self._system_summary is None:
# Note there is no attempt to canonicalize these values. If two
# machines that would in fact produce identical builds differ in
# these values, it just means that a superfluous build will be
# done once to get the mapping from new input hash to preexisting
# output hash into the cache.
assert len(sys.platform) != 0, len(platform.machine()) != 0
# Use environment from command so we can access MinGW on windows.
env = command.PlatformEnvironment([])
gcc = file_tools.Which('gcc', paths=env['PATH'].split(os.pathsep))
p = subprocess.Popen([gcc, '-v'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
_, gcc_version = p.communicate()
assert p.returncode == 0
items = [
('platform', sys.platform),
('machine', platform.machine()),
('gcc-v', gcc_version),
]
self._system_summary = str(items)
return self._system_summary
def Invoke(self,
check_call,
package,
inputs,
output,
cwd,
build_signature=None):
# TODO(bradnelson): Instead of allowing full subprocess functionality,
# move execution here and use polymorphism to implement things like
# mkdir, copy directly in python.
kwargs = self._kwargs.copy()
kwargs['cwd'] = os.path.join(os.path.abspath(cwd),
kwargs.get('cwd', '.'))
values = PrepareCommandValues(kwargs['cwd'], inputs, output)
try:
values['cores'] = multiprocessing.cpu_count()
except NotImplementedError:
values['cores'] = 4 # Assume 4 if we can't measure.
values['package'] = package
if build_signature is not None:
values['build_signature'] = build_signature
values['top_srcdir'] = FixPath(os.path.relpath(NACL_DIR,
kwargs['cwd']))
values['abs_top_srcdir'] = FixPath(os.path.abspath(NACL_DIR))
# Use mingw on windows.
if sys.platform == 'win32':
# TODO(bradnelson): switch to something hermetic.
mingw = os.environ.get('MINGW', r'c:\mingw')
msys = os.path.join(mingw, 'msys', '1.0')
if not os.path.exists(msys):
msys = os.path.join(mingw, 'msys')
# We need both msys (posix like build environment) and MinGW (windows
# build of tools like gcc). We add <MINGW>/msys/[1.0/]bin to the path to
# get sh.exe. We also add an msys style path (/mingw/bin) to get things
# like gcc from inside msys.
kwargs['path_dirs'] = (
['/mingw/bin', os.path.join(msys, 'bin')] +
kwargs.get('path_dirs', []))
if 'path_dirs' in kwargs:
path_dirs = [dirname % values for dirname in kwargs['path_dirs']]
del kwargs['path_dirs']
env = os.environ.copy()
env['PATH'] = os.pathsep.join(path_dirs +
env['PATH'].split(os.pathsep))
kwargs['env'] = env
if isinstance(self._command, str):
command = self._command % values
else:
command = [arg % values for arg in self._command]
paths = kwargs.get('env', os.environ).get('PATH',
'').split(os.pathsep)
command[0] = file_tools.Which(command[0], paths=paths)
check_call(command, **kwargs)