def _cleanup_old_layouts(using_kitchen, properties, buildbot_build_dir,
cache_dir):
cleanup_paths = []
# Make switching to remote_run easier: we do not use buildbot workdir,
# and it takes disk space leading to out of disk errors.
buildbot_workdir = properties.get('workdir')
if buildbot_workdir and os.path.isdir(buildbot_workdir):
try:
buildbot_workdir = os.path.realpath(buildbot_workdir)
buildbot_build_dir = os.path.realpath(buildbot_build_dir)
if buildbot_build_dir.startswith(buildbot_workdir):
buildbot_workdir = buildbot_build_dir
# Buildbot workdir is usually used as current working directory,
# so do not remove it, but delete all of the contents. Deleting
# current working directory of a running process may cause
# confusing errors.
cleanup_paths.extend(
os.path.join(buildbot_workdir, x)
for x in os.listdir(buildbot_workdir))
except Exception:
# It's preferred that we keep going rather than fail the build
# on optional cleanup.
LOGGER.exception('Buildbot workdir cleanup failed: %s',
buildbot_workdir)
if using_kitchen:
# We want to delete the 'git_cache' cache directory, which was used by the
# legacy "remote" code path.
cleanup_paths.append(os.path.join(cache_dir, 'git_cache'))
# We want to delete "<cache>/b", the legacy build cache directory. We will
# use "<cache>/builder" from the "generic" infra configuration setup.
cleanup_paths.append(os.path.join(cache_dir, 'b'))
else:
# If we have a 'git' cache directory from a previous Kitchen run, we should
# delete that in favor of the 'git_cache' cache directory.
cleanup_paths.append(os.path.join(cache_dir, 'git'))
# We want to delete "<cache>/builder" from the Kitchen run.
cleanup_paths.append(os.path.join(cache_dir, 'builder'))
cleanup_paths = [p for p in cleanup_paths if os.path.exists(p)]
if cleanup_paths:
LOGGER.info('Cleaning up %d old layout path(s)...', len(cleanup_paths))
for path in cleanup_paths:
LOGGER.info('Removing path from previous layout: %s', path)
try:
chromium_utils.RemovePath(path)
except Exception:
LOGGER.exception('Failed to cleanup path: %s', path)
def remove_temp():
"""Removes all the temp files on Windows."""
with function_logger('removing TEMP'):
root = os.environ['TEMP']
for path in os.listdir(root):
if path.startswith('goma'):
# Work around http://crbug.com/449511
continue
try:
chromium_utils.RemovePath(os.path.join(root, path))
except OSError:
pass
def main(argv):
parser = argparse.ArgumentParser()
parser.add_argument('--repository',
required=True,
help='URL of a git repository to fetch.')
parser.add_argument('--revision', help='Git commit hash to check out.')
parser.add_argument('--recipe',
required=True,
help='Name of the recipe to run')
parser.add_argument('--build-properties-gz',
dest='build_properties',
type=chromium_utils.convert_gz_json_type,
default={},
help='Build properties in b64 gz JSON format')
parser.add_argument('--factory-properties-gz',
dest='factory_properties',
type=chromium_utils.convert_gz_json_type,
default={},
help='factory properties in b64 gz JSON format')
parser.add_argument('--leak',
action='store_true',
help='Refrain from cleaning up generated artifacts.')
parser.add_argument('--verbose', action='store_true')
group = parser.add_argument_group('LogDog Bootstrap')
logdog_bootstrap.add_arguments(group)
args = parser.parse_args(argv[1:])
with robust_tempdir.RobustTempdir(prefix='rr', leak=args.leak) as rt:
try:
basedir = chromium_utils.FindUpward(os.getcwd(), 'b')
except chromium_utils.PathNotFound as e:
LOGGER.warn(e)
# Use base directory inside system temporary directory - if we use slave
# one (cwd), the paths get too long. Recipes which need different paths
# or persistent directories should do so explicitly.
basedir = tempfile.gettempdir()
# Explicitly clean up possibly leaked temporary directories
# from previous runs.
rt.cleanup(basedir)
tempdir = rt.tempdir(basedir)
LOGGER.info('Using temporary directory: [%s].', tempdir)
build_data_dir = rt.tempdir(basedir)
LOGGER.info('Using build data directory: [%s].', build_data_dir)
properties = copy.copy(args.factory_properties)
properties.update(args.build_properties)
properties['build_data_dir'] = build_data_dir
LOGGER.info('Using properties: %r', properties)
properties_file = os.path.join(tempdir, 'remote_run_properties.json')
with open(properties_file, 'w') as f:
json.dump(properties, f)
monitoring_utils.write_build_monitoring_event(build_data_dir,
properties)
# Make switching to remote_run easier: we do not use buildbot workdir,
# and it takes disk space leading to out of disk errors.
buildbot_workdir = properties.get('workdir')
if buildbot_workdir:
try:
if os.path.exists(buildbot_workdir):
buildbot_workdir = os.path.realpath(buildbot_workdir)
cwd = os.path.realpath(os.getcwd())
if cwd.startswith(buildbot_workdir):
buildbot_workdir = cwd
LOGGER.info('Cleaning up buildbot workdir %r',
buildbot_workdir)
# Buildbot workdir is usually used as current working directory,
# so do not remove it, but delete all of the contents. Deleting
# current working directory of a running process may cause
# confusing errors.
for p in (os.path.join(buildbot_workdir, x)
for x in os.listdir(buildbot_workdir)):
LOGGER.info('Deleting %r', p)
chromium_utils.RemovePath(p)
except Exception as e:
# It's preferred that we keep going rather than fail the build
# on optional cleanup.
LOGGER.exception('Buildbot workdir cleanup failed: %s', e)
# Should we use a CIPD pin?
mastername = properties.get('mastername')
cipd_pin = None
if mastername:
cipd_pin = _CIPD_PINS.get(mastername)
if not cipd_pin:
cipd_pin = _CIPD_PINS[None]
cipd_path = os.path.join(basedir, '.remote_run_cipd')
_install_cipd_packages(cipd_path,
cipd.CipdPackage('infra/recipes-py', cipd_pin))
recipe_result_path = os.path.join(tempdir, 'recipe_result.json')
recipe_cmd = [
sys.executable,
os.path.join(cipd_path, 'recipes.py'),
'--verbose',
'remote',
'--repository',
args.repository,
'--revision',
args.revision,
'--workdir',
os.path.join(tempdir, 'rw'),
'--',
'--verbose',
'run',
'--properties-file',
properties_file,
'--workdir',
os.path.join(tempdir, 'w'),
'--output-result-json',
recipe_result_path,
args.recipe,
]
# If we bootstrap through logdog, the recipe command line gets written
# to a temporary file and does not appear in the log.
LOGGER.info('Recipe command line: %r', recipe_cmd)
recipe_return_code = None
try:
bs = logdog_bootstrap.bootstrap(rt, args, basedir, tempdir,
properties, recipe_cmd)
LOGGER.info('Bootstrapping through LogDog: %s', bs.cmd)
_ = _call(bs.cmd)
recipe_return_code = bs.get_result()
except logdog_bootstrap.NotBootstrapped as e:
LOGGER.info('Not bootstrapped: %s', e.message)
except logdog_bootstrap.BootstrapError as e:
LOGGER.warning('Could not bootstrap LogDog: %s', e.message)
except Exception as e:
LOGGER.exception('Exception while bootstrapping LogDog.')
finally:
if recipe_return_code is None:
LOGGER.info(
'Not using LogDog. Invoking `recipes.py` directly.')
recipe_return_code = _call(recipe_cmd)
# Try to open recipe result JSON. Any failure will result in an exception
# and an infra failure.
with open(recipe_result_path) as f:
json.load(f)
return recipe_return_code