def Update(config, active_master, c):
factory = BuildFactory()
factory_commands = commands.FactoryCommands(factory)
factory_commands.AddUpdateScriptStep()
# pylint: disable=W0212
gitzip_exe = os.path.join(factory_commands._script_dir, 'gitzip.py')
cmd = [
'python', gitzip_exe, '--workdir', '.', '--url',
'%schromium/src.git' % config.Master.git_server_url, '--gs_bucket',
'gs://chromium-git-bundles', '--gs_acl', 'public-read', '--timeout',
'%d' % (60 * 60), '--stayalive', '200', '--verbose'
]
factory.addStep(shell.ShellCommand,
name='gitzip',
description='gitzip',
timeout=7200,
workdir='',
command=cmd)
builders = c.setdefault('builders', [])
builders.append({
'name': 'Chromium Git Packager',
'builddir': 'chromium-git-packager',
'factory': factory,
'auto_reboot': False
})
schedulers = c.setdefault('schedulers', [])
schedulers.append(
Nightly(name='gitzip_nightly',
branch=None,
builderNames=['Chromium Git Packager'],
hour=2,
minute=40))
def AddUpdateStep(self,
gclient_spec,
factory_properties,
factory,
slave_type,
sudo_for_remove=False,
gclient_deps=None,
options=None,
blink_config=False):
if gclient_spec is None:
gclient_spec = self.BuildGClientSpec()
factory_properties = factory_properties or {}
# Get the factory command object to add update step to the factory.
factory_cmd_obj = commands.FactoryCommands(
factory, target_platform=self._target_platform)
# Get variables needed for the update.
env = factory_properties.get('gclient_env', {})
timeout = factory_properties.get('gclient_timeout')
no_gclient_branch = factory_properties.get('no_gclient_branch', False)
no_gclient_revision = factory_properties.get('no_gclient_revision',
False)
gclient_transitive = factory_properties.get('gclient_transitive',
False)
primary_repo = factory_properties.get('primary_repo', '')
gclient_jobs = factory_properties.get('gclient_jobs')
# Do not run gyp_chromium on testers.
if slave_type in ('Tester', ):
env.update({'GYP_CHROMIUM_NO_ACTION': '1'})
# Add the update step.
factory_cmd_obj.AddUpdateStep(gclient_spec,
env=env,
timeout=timeout,
sudo_for_remove=sudo_for_remove,
gclient_deps=gclient_deps,
gclient_nohooks=True,
no_gclient_branch=no_gclient_branch,
no_gclient_revision=no_gclient_revision,
gclient_transitive=gclient_transitive,
primary_repo=primary_repo,
gclient_jobs=gclient_jobs,
blink_config=blink_config)
if slave_type in ('AnnotatedTrybot', 'CrosTrybot', 'Trybot', 'Bisect',
'TrybotTester', 'TrybotBuilder'):
factory_cmd_obj.AddApplyIssueStep(
timeout=timeout,
server=config.Master.Master4.code_review_site,
revision_mapping=self._revision_mapping)
if not self._nohooks_on_update:
factory_cmd_obj.AddRunHooksStep(env=env,
timeout=timeout,
options=options)
def BaseFactory(self, gclient_spec=None, official_release=False,
factory_properties=None, build_properties=None,
delay_compile_step=False, sudo_for_remove=False,
gclient_deps=None, slave_type=None):
if gclient_spec is None:
gclient_spec = self.BuildGClientSpec()
factory_properties = factory_properties or {}
factory = BuildFactory(build_properties)
factory_cmd_obj = commands.FactoryCommands(factory,
target_platform=self._target_platform)
# First kill any svn.exe tasks so we can update in peace, and
# afterwards use the checked-out script to kill everything else.
if (self._target_platform == 'win32' and
not factory_properties.get('no_kill')):
factory_cmd_obj.AddSvnKillStep()
script_solutions = None
if (factory_properties.get('goma_canary') and
factory_properties.get('slave_internal_url')):
script_solutions = [GClientSolution(
factory_properties.get('slave_internal_url'),
name="slave.DEPS",
custom_vars_list=[("goma_linux_revision", "HEAD"),
("goma_mac_revision", "HEAD"),
("goma_win_revision", "HEAD")])]
factory_cmd_obj.AddUpdateScriptStep(
gclient_jobs=factory_properties.get('update_scripts_gclient_jobs'),
solutions=script_solutions)
# Once the script is updated, the zombie processes left by the previous
# run can be killed.
if (self._target_platform == 'win32' and
not factory_properties.get('no_kill')):
factory_cmd_obj.AddTaskkillStep()
# Revert the tree to a clean (unmodified) state.
factory_cmd_obj.AddGClientRevertStep()
env = factory_properties.get('gclient_env', {})
# Allow gclient_deps to also come from the factory_properties.
if gclient_deps == None:
gclient_deps = factory_properties.get('gclient_deps', None)
if gclient_deps == 'ios':
gclient_spec += ';target_os = [\'ios\'];target_os_only = True'
# svn timeout is 2 min; we allow 5
timeout = factory_properties.get('gclient_timeout')
if official_release or factory_properties.get('nuke_and_pave'):
no_gclient_branch = factory_properties.get('no_gclient_branch', False)
factory_cmd_obj.AddClobberTreeStep(gclient_spec, env, timeout,
gclient_deps=gclient_deps, gclient_nohooks=self._nohooks_on_update,
no_gclient_branch=no_gclient_branch)
elif not delay_compile_step:
self.AddUpdateStep(gclient_spec, factory_properties, factory,
slave_type, sudo_for_remove, gclient_deps=gclient_deps)
return factory
def BuildFactory(self, target='Release', clobber=False, tests=None, mode=None,
slave_type='BuilderTester', options=None,
compile_timeout=1200, build_url=None, project=None,
factory_properties=None, gclient_deps=None,
target_arch=None):
factory_properties = factory_properties or {}
if (options and
'--build-tool=ninja' in options and '--compiler=goma-clang' in options):
# Ninja needs CC and CXX set at gyp time.
factory_properties['gclient_env']['CC'] = 'clang'
factory_properties['gclient_env']['CXX'] = 'clang++'
# Create the spec for the solutions
gclient_spec = self.BuildGClientSpec(tests)
# Initialize the factory with the basic steps.
factory = self.BaseFactory(gclient_spec,
factory_properties=factory_properties,
slave_type=slave_type,
gclient_deps=gclient_deps)
# Get the factory command object to create new steps to the factory.
factory_cmd_obj = commands.FactoryCommands(factory, target,
self._build_dir,
self._target_platform,
target_arch)
# Update clang if necessary.
gclient_env = factory_properties.get('gclient_env', {})
if ('clang=1' in gclient_env.get('GYP_DEFINES', '') or
'asan=1' in gclient_env.get('GYP_DEFINES', '')):
factory_cmd_obj.AddUpdateClangStep()
# Add a step to cleanup temporary files and data left from a previous run
# to prevent the drives from becoming full over time.
factory_cmd_obj.AddTempCleanupStep()
# Add the compile step if needed.
if slave_type in ['BuilderTester', 'Builder', 'Trybot', 'Indexer']:
factory_cmd_obj.AddCompileStep(project or self._project, clobber,
mode=mode, options=options,
timeout=compile_timeout)
# Archive the full output directory if the machine is a builder.
if slave_type == 'Builder':
factory_cmd_obj.AddZipBuild(halt_on_failure=True,
factory_properties=factory_properties)
# Download the full output directory if the machine is a tester.
if slave_type == 'Tester':
factory_cmd_obj.AddExtractBuild(build_url,
factory_properties=factory_properties)
return factory
def AddUpdateStep(self, gclient_spec, factory_properties, factory,
slave_type, sudo_for_remove=False, gclient_deps=None):
if gclient_spec is None:
gclient_spec = self.BuildGClientSpec()
factory_properties = factory_properties or {}
# Get the factory command object to add update step to the factory.
factory_cmd_obj = commands.FactoryCommands(factory,
target_platform=self._target_platform)
# Get variables needed for the update.
env = factory_properties.get('gclient_env', {})
timeout = factory_properties.get('gclient_timeout')
no_gclient_branch = factory_properties.get('no_gclient_branch', False)
gclient_transitive = factory_properties.get('gclient_transitive', False)
primary_repo = factory_properties.get('primary_repo', '')
gclient_jobs = factory_properties.get('gclient_jobs')
# Add the update step.
factory_cmd_obj.AddUpdateStep(
gclient_spec,
env=env,
timeout=timeout,
sudo_for_remove=sudo_for_remove,
gclient_deps=gclient_deps,
gclient_nohooks=True,
no_gclient_branch=no_gclient_branch,
gclient_transitive=gclient_transitive,
primary_repo=primary_repo,
gclient_jobs=gclient_jobs)
if slave_type == 'Trybot':
factory_cmd_obj.AddApplyIssueStep()
if not self._nohooks_on_update:
factory_cmd_obj.AddRunHooksStep(env=env, timeout=timeout)
def BaseFactory(self, gclient_spec=None, official_release=False,
factory_properties=None, build_properties=None,
delay_compile_step=False, sudo_for_remove=False,
gclient_deps=None, slave_type=None):
if gclient_spec is None:
gclient_spec = self.BuildGClientSpec()
factory_properties = factory_properties or {}
factory = BuildFactory(build_properties)
factory_cmd_obj = commands.FactoryCommands(factory,
target_platform=self._target_platform)
# First kill any svn.exe tasks so we can update in peace, and
# afterwards use the checked-out script to kill everything else.
if (self._target_platform == 'win32' and
not factory_properties.get('no_kill')):
factory_cmd_obj.AddSvnKillStep()
factory_cmd_obj.AddUpdateScriptStep(
gclient_jobs=factory_properties.get('update_scripts_gclient_jobs'))
# Once the script is updated, the zombie processes left by the previous
# run can be killed.
if (self._target_platform == 'win32' and
not factory_properties.get('no_kill')):
factory_cmd_obj.AddTaskkillStep()
env = factory_properties.get('gclient_env', {})
# Allow gclient_deps to also come from the factory_properties.
if gclient_deps == None:
gclient_deps = factory_properties.get('gclient_deps', None)
# svn timeout is 2 min; we allow 5
timeout = factory_properties.get('gclient_timeout')
if official_release or factory_properties.get('nuke_and_pave'):
no_gclient_branch = factory_properties.get('no_gclient_branch', False)
factory_cmd_obj.AddClobberTreeStep(gclient_spec, env, timeout,
gclient_deps=gclient_deps, gclient_nohooks=self._nohooks_on_update,
no_gclient_branch=no_gclient_branch)
elif not delay_compile_step:
self.AddUpdateStep(gclient_spec, factory_properties, factory,
slave_type, sudo_for_remove, gclient_deps=gclient_deps)
return factory
def BuildFactory(self,
target='Release',
clobber=False,
tests=None,
mode=None,
slave_type='BuilderTester',
options=None,
compile_timeout=1200,
build_url=None,
project=None,
factory_properties=None,
gclient_deps=None,
target_arch=None,
skip_archive_steps=False):
if factory_properties is None:
factory_properties = {}
factory_properties.setdefault('gclient_env', {})
gclient_env = factory_properties['gclient_env']
# Create the spec for the solutions
gclient_spec = self.BuildGClientSpec(tests)
# Initialize the factory with the basic steps.
factory = self.BaseFactory(gclient_spec,
factory_properties=factory_properties,
slave_type=slave_type,
gclient_deps=gclient_deps,
options=options)
# Optional repository root (default: 'src').
repository_root = factory_properties.get('repository_root', 'src')
# Get the factory command object to create new steps to the factory.
factory_cmd_obj = commands.FactoryCommands(factory, target,
self._build_dir,
self._target_platform,
target_arch,
repository_root)
# Update clang if necessary.
if ('clang=1' in gclient_env.get('GYP_DEFINES', '')
or (self._target_platform != 'win32'
and factory_properties.get('asan'))):
factory_cmd_obj.AddUpdateClangStep()
# Add a step to cleanup temporary files and data left from a previous run
# to prevent the drives from becoming full over time.
factory_cmd_obj.AddTempCleanupStep()
# Update the NaCl SDK if needed
if factory_properties.get('update_nacl_sdk'):
factory_cmd_obj.AddUpdateNaClSDKStep(
factory_properties['update_nacl_sdk'])
# Add the compile step if needed.
if slave_type in [
'BuilderTester', 'Builder', 'Trybot', 'Indexer',
'TrybotBuilder'
]:
# If we want to confirm that two successive compiles result in a no-op
# build, tell the compile step to verify that.
if factory_properties.get('confirm_noop_compile'):
options = ['--ninja-ensure-up-to-date'] + options
factory_cmd_obj.AddCompileStep(
project,
clobber,
mode=mode,
options=options,
timeout=compile_timeout,
env=factory_properties.get('compile_env'))
# Generate synthetic user profiles. Must run before AddZipBuild().
if factory_properties.get('create_profiles'):
# pylint: disable=W0212
factory_cmd_obj.AddProfileCreationStep('small_profile')
if not skip_archive_steps:
# Archive the full output directory if the machine is a builder.
if slave_type in ['Builder', 'TrybotBuilder']:
if build_url or factory_properties.get('build_url'):
# There are some builders that are classified as builders, but only
# because we only need to see if the patch compiles, and not
# actually because triggered testers needs the build. So if we don't
# supply a build_url, we assume that the build artifact doesn't
# need to be uploaded and we can bypass this step.
factory_cmd_obj.AddZipBuild(
build_url, factory_properties=factory_properties)
# Download the full output directory if the machine is a tester.
if slave_type in ['Tester', 'TrybotTester']:
factory_cmd_obj.AddExtractBuild()
return factory
def BaseFactory(self,
gclient_spec=None,
official_release=False,
factory_properties=None,
build_properties=None,
sudo_for_remove=False,
gclient_deps=None,
slave_type=None,
options=None,
target=None):
if gclient_spec is None:
gclient_spec = self.BuildGClientSpec()
factory_properties = factory_properties or {}
factory = BuildFactory(build_properties)
factory_cmd_obj = commands.FactoryCommands(
factory, target_platform=self._target_platform)
# First kill any svn.exe tasks so we can update in peace, and
# afterwards use the checked-out script to kill everything else.
if (self._target_platform == 'win32'
and not factory_properties.get('no_kill')):
factory_cmd_obj.AddSvnKillStep()
extra_args = None
if factory_properties.get('goma_canary'):
extra_args = ['--revision', 'build/goma@HEAD']
factory_cmd_obj.AddUpdateScriptStep(
gclient_jobs=factory_properties.get('update_scripts_gclient_jobs'),
args=extra_args)
# Once the script is updated, the zombie processes left by the previous
# run can be killed.
if (self._target_platform == 'win32'
and not factory_properties.get('no_kill')):
factory_cmd_obj.AddTaskkillStep()
env = factory_properties.get('gclient_env', {})
# Allow gclient_deps to also come from the factory_properties.
if gclient_deps == None:
gclient_deps = factory_properties.get('gclient_deps', None)
if gclient_deps == 'ios':
gclient_spec += ';target_os = [\'ios\'];target_os_only = True'
# Do we need to operate in magic blink mode?
blink_config = factory_properties.get('blink_config')
# Force the build checkout to be at some revision. This may or may not
# activate depending on its own criteria, but the expectation is that if
# this does activate, it will emit a BOT_UPDATED file in the build/
# directory to signal to the other gclient update steps to no-op.
code_review_site = config.Master.Master4.code_review_site
factory_cmd_obj.AddBotUpdateStep(env,
gclient_spec,
self._revision_mapping,
server=code_review_site,
blink_config=blink_config)
use_mb = factory_properties.get('use_mb')
if use_mb:
# Don't run GYP as part of gclient runhooks.
env.update({'GYP_CHROMIUM_NO_ACTION': '1'})
# svn timeout is 2 min; we allow 5
timeout = factory_properties.get('gclient_timeout')
if official_release or factory_properties.get('nuke_and_pave'):
no_gclient_branch = factory_properties.get('no_gclient_branch',
False)
factory_cmd_obj.AddClobberTreeStep(
gclient_spec,
env,
timeout,
gclient_deps=gclient_deps,
gclient_nohooks=self._nohooks_on_update,
no_gclient_branch=no_gclient_branch,
options=options)
else:
# Revert the tree to a clean (unmodified) state.
factory_cmd_obj.AddGClientRevertStep()
self.AddUpdateStep(gclient_spec,
factory_properties,
factory,
slave_type,
sudo_for_remove,
gclient_deps=gclient_deps,
options=options,
blink_config=blink_config)
if use_mb:
# To be safe, we reset env back to its non-MB state and then make sure
# MB is passed an env w/ no GYP flags set at all.
del env['GYP_CHROMIUM_NO_ACTION']
mb_env = env.copy()
if 'GYP_DEFINES' in mb_env:
del mb_env['GYP_DEFINES']
if 'GYP_CROSSCOMPILE' in mb_env:
del mb_env['GYP_CROSSCOMPILE']
factory_cmd_obj.AddGenerateBuildFilesStep(env=mb_env,
timeout=timeout,
options=options,
target=target)
return factory
def BuildFactory(self,
target='Release',
clobber=False,
tests=None,
mode=None,
slave_type='BuilderTester',
options=None,
compile_timeout=1200,
build_url=None,
project=None,
factory_properties=None,
gclient_deps=None,
target_arch=None,
skip_archive_steps=False):
if factory_properties is None:
factory_properties = {}
factory_properties.setdefault('gclient_env', {})
gclient_env = factory_properties['gclient_env']
if options and '--build-tool=ninja' in options:
gclient_env['GYP_GENERATORS'] = 'ninja'
# Create the spec for the solutions
gclient_spec = self.BuildGClientSpec(tests)
# Initialize the factory with the basic steps.
factory = self.BaseFactory(gclient_spec,
factory_properties=factory_properties,
slave_type=slave_type,
gclient_deps=gclient_deps,
options=options)
# Optional repository root (default: 'src').
repository_root = factory_properties.get('repository_root', 'src')
# Get the factory command object to create new steps to the factory.
factory_cmd_obj = commands.FactoryCommands(factory, target,
self._build_dir,
self._target_platform,
target_arch,
repository_root)
# Update clang if necessary.
if ('clang=1' in gclient_env.get('GYP_DEFINES', '')
or (self._target_platform != 'win32'
and factory_properties.get('asan'))):
factory_cmd_obj.AddUpdateClangStep()
# Add a step to cleanup temporary files and data left from a previous run
# to prevent the drives from becoming full over time.
factory_cmd_obj.AddTempCleanupStep()
# Update the NaCl SDK if needed
if factory_properties.get('update_nacl_sdk'):
factory_cmd_obj.AddUpdateNaClSDKStep(
factory_properties['update_nacl_sdk'])
# Add the compile step if needed.
if slave_type in [
'BuilderTester', 'Builder', 'Trybot', 'Indexer',
'TrybotBuilder'
]:
if self._target_platform == 'win32':
# Look for a solution named for its enclosing directory.
project = project or os.path.basename(self._build_dir) + '.sln'
factory_cmd_obj.AddCompileStep(
project,
clobber,
mode=mode,
options=options,
timeout=compile_timeout,
env=factory_properties.get('compile_env'))
# Generate synthetic user profiles. Must run before AddZipBuild().
if factory_properties.get('create_profiles'):
# pylint: disable=W0212
factory_cmd_obj.AddProfileCreationStep('small_profile')
if not skip_archive_steps:
# Archive the full output directory if the machine is a builder.
if slave_type in ['Builder', 'TrybotBuilder']:
factory_cmd_obj.AddZipBuild(
halt_on_failure=True,
factory_properties=factory_properties)
# Download the full output directory if the machine is a tester.
if slave_type in ['Tester', 'TrybotTester']:
factory_cmd_obj.AddExtractBuild(
build_url, factory_properties=factory_properties)
return factory
def BaseFactory(self,
gclient_spec=None,
official_release=False,
factory_properties=None,
build_properties=None,
delay_compile_step=False,
sudo_for_remove=False,
gclient_deps=None,
slave_type=None,
options=None):
if gclient_spec is None:
gclient_spec = self.BuildGClientSpec()
factory_properties = factory_properties or {}
factory = BuildFactory(build_properties)
factory_cmd_obj = commands.FactoryCommands(
factory, target_platform=self._target_platform)
# First kill any svn.exe tasks so we can update in peace, and
# afterwards use the checked-out script to kill everything else.
if (self._target_platform == 'win32'
and not factory_properties.get('no_kill')):
factory_cmd_obj.AddSvnKillStep()
script_solutions = None
if (factory_properties.get('goma_canary')
and factory_properties.get('slave_internal_url')):
script_solutions = [
GClientSolution(factory_properties.get('slave_internal_url'),
name="slave.DEPS",
custom_vars_list=[
("goma_linux_revision", "HEAD"),
("goma_mac_revision", "HEAD"),
("goma_win_revision", "HEAD")
])
]
factory_cmd_obj.AddUpdateScriptStep(
gclient_jobs=factory_properties.get('update_scripts_gclient_jobs'),
solutions=script_solutions)
# Once the script is updated, the zombie processes left by the previous
# run can be killed.
if (self._target_platform == 'win32'
and not factory_properties.get('no_kill')):
factory_cmd_obj.AddTaskkillStep()
env = factory_properties.get('gclient_env', {})
# Allow gclient_deps to also come from the factory_properties.
if gclient_deps == None:
gclient_deps = factory_properties.get('gclient_deps', None)
if gclient_deps == 'ios':
gclient_spec += ';target_os = [\'ios\'];target_os_only = True'
# Force the build checkout to be at some revision. This may or may not
# activate depending on its own criteria, but the expectation is that if
# this does activate, it will emit a BOT_UPDATED file in the build/
# directory to signal to the other gclient update steps to no-op.
code_review_site = config.Master.Master4.code_review_site
factory_cmd_obj.AddBotUpdateStep(env,
gclient_spec,
self._revision_mapping,
server=code_review_site)
# svn timeout is 2 min; we allow 5
timeout = factory_properties.get('gclient_timeout')
if official_release or factory_properties.get('nuke_and_pave'):
no_gclient_branch = factory_properties.get('no_gclient_branch',
False)
factory_cmd_obj.AddClobberTreeStep(
gclient_spec,
env,
timeout,
gclient_deps=gclient_deps,
gclient_nohooks=self._nohooks_on_update,
no_gclient_branch=no_gclient_branch,
options=options)
else:
# Revert the tree to a clean (unmodified) state.
factory_cmd_obj.AddGClientRevertStep()
if not delay_compile_step:
self.AddUpdateStep(gclient_spec,
factory_properties,
factory,
slave_type,
sudo_for_remove,
gclient_deps=gclient_deps,
options=options)
return factory