def Update(config, active_master, c): factory = BuildFactory() factory_commands = commands.FactoryCommands(factory) factory_commands.AddUpdateScriptStep() # pylint: disable=W0212 gitzip_exe = os.path.join(factory_commands._script_dir, 'gitzip.py') cmd = [ 'python', gitzip_exe, '--workdir', '.', '--url', '%schromium/src.git' % config.Master.git_server_url, '--gs_bucket', 'gs://chromium-git-bundles', '--gs_acl', 'public-read', '--timeout', '%d' % (60 * 60), '--stayalive', '200', '--verbose' ] factory.addStep(shell.ShellCommand, name='gitzip', description='gitzip', timeout=7200, workdir='', command=cmd) builders = c.setdefault('builders', []) builders.append({ 'name': 'Chromium Git Packager', 'builddir': 'chromium-git-packager', 'factory': factory, 'auto_reboot': False }) schedulers = c.setdefault('schedulers', []) schedulers.append( Nightly(name='gitzip_nightly', branch=None, builderNames=['Chromium Git Packager'], hour=2, minute=40))
def AddUpdateStep(self, gclient_spec, factory_properties, factory, slave_type, sudo_for_remove=False, gclient_deps=None, options=None, blink_config=False): if gclient_spec is None: gclient_spec = self.BuildGClientSpec() factory_properties = factory_properties or {} # Get the factory command object to add update step to the factory. factory_cmd_obj = commands.FactoryCommands( factory, target_platform=self._target_platform) # Get variables needed for the update. env = factory_properties.get('gclient_env', {}) timeout = factory_properties.get('gclient_timeout') no_gclient_branch = factory_properties.get('no_gclient_branch', False) no_gclient_revision = factory_properties.get('no_gclient_revision', False) gclient_transitive = factory_properties.get('gclient_transitive', False) primary_repo = factory_properties.get('primary_repo', '') gclient_jobs = factory_properties.get('gclient_jobs') # Do not run gyp_chromium on testers. if slave_type in ('Tester', ): env.update({'GYP_CHROMIUM_NO_ACTION': '1'}) # Add the update step. factory_cmd_obj.AddUpdateStep(gclient_spec, env=env, timeout=timeout, sudo_for_remove=sudo_for_remove, gclient_deps=gclient_deps, gclient_nohooks=True, no_gclient_branch=no_gclient_branch, no_gclient_revision=no_gclient_revision, gclient_transitive=gclient_transitive, primary_repo=primary_repo, gclient_jobs=gclient_jobs, blink_config=blink_config) if slave_type in ('AnnotatedTrybot', 'CrosTrybot', 'Trybot', 'Bisect', 'TrybotTester', 'TrybotBuilder'): factory_cmd_obj.AddApplyIssueStep( timeout=timeout, server=config.Master.Master4.code_review_site, revision_mapping=self._revision_mapping) if not self._nohooks_on_update: factory_cmd_obj.AddRunHooksStep(env=env, timeout=timeout, options=options)
def BaseFactory(self, gclient_spec=None, official_release=False, factory_properties=None, build_properties=None, delay_compile_step=False, sudo_for_remove=False, gclient_deps=None, slave_type=None): if gclient_spec is None: gclient_spec = self.BuildGClientSpec() factory_properties = factory_properties or {} factory = BuildFactory(build_properties) factory_cmd_obj = commands.FactoryCommands(factory, target_platform=self._target_platform) # First kill any svn.exe tasks so we can update in peace, and # afterwards use the checked-out script to kill everything else. if (self._target_platform == 'win32' and not factory_properties.get('no_kill')): factory_cmd_obj.AddSvnKillStep() script_solutions = None if (factory_properties.get('goma_canary') and factory_properties.get('slave_internal_url')): script_solutions = [GClientSolution( factory_properties.get('slave_internal_url'), name="slave.DEPS", custom_vars_list=[("goma_linux_revision", "HEAD"), ("goma_mac_revision", "HEAD"), ("goma_win_revision", "HEAD")])] factory_cmd_obj.AddUpdateScriptStep( gclient_jobs=factory_properties.get('update_scripts_gclient_jobs'), solutions=script_solutions) # Once the script is updated, the zombie processes left by the previous # run can be killed. if (self._target_platform == 'win32' and not factory_properties.get('no_kill')): factory_cmd_obj.AddTaskkillStep() # Revert the tree to a clean (unmodified) state. factory_cmd_obj.AddGClientRevertStep() env = factory_properties.get('gclient_env', {}) # Allow gclient_deps to also come from the factory_properties. if gclient_deps == None: gclient_deps = factory_properties.get('gclient_deps', None) if gclient_deps == 'ios': gclient_spec += ';target_os = [\'ios\'];target_os_only = True' # svn timeout is 2 min; we allow 5 timeout = factory_properties.get('gclient_timeout') if official_release or factory_properties.get('nuke_and_pave'): no_gclient_branch = factory_properties.get('no_gclient_branch', False) factory_cmd_obj.AddClobberTreeStep(gclient_spec, env, timeout, gclient_deps=gclient_deps, gclient_nohooks=self._nohooks_on_update, no_gclient_branch=no_gclient_branch) elif not delay_compile_step: self.AddUpdateStep(gclient_spec, factory_properties, factory, slave_type, sudo_for_remove, gclient_deps=gclient_deps) return factory
def BuildFactory(self, target='Release', clobber=False, tests=None, mode=None, slave_type='BuilderTester', options=None, compile_timeout=1200, build_url=None, project=None, factory_properties=None, gclient_deps=None, target_arch=None): factory_properties = factory_properties or {} if (options and '--build-tool=ninja' in options and '--compiler=goma-clang' in options): # Ninja needs CC and CXX set at gyp time. factory_properties['gclient_env']['CC'] = 'clang' factory_properties['gclient_env']['CXX'] = 'clang++' # Create the spec for the solutions gclient_spec = self.BuildGClientSpec(tests) # Initialize the factory with the basic steps. factory = self.BaseFactory(gclient_spec, factory_properties=factory_properties, slave_type=slave_type, gclient_deps=gclient_deps) # Get the factory command object to create new steps to the factory. factory_cmd_obj = commands.FactoryCommands(factory, target, self._build_dir, self._target_platform, target_arch) # Update clang if necessary. gclient_env = factory_properties.get('gclient_env', {}) if ('clang=1' in gclient_env.get('GYP_DEFINES', '') or 'asan=1' in gclient_env.get('GYP_DEFINES', '')): factory_cmd_obj.AddUpdateClangStep() # Add a step to cleanup temporary files and data left from a previous run # to prevent the drives from becoming full over time. factory_cmd_obj.AddTempCleanupStep() # Add the compile step if needed. if slave_type in ['BuilderTester', 'Builder', 'Trybot', 'Indexer']: factory_cmd_obj.AddCompileStep(project or self._project, clobber, mode=mode, options=options, timeout=compile_timeout) # Archive the full output directory if the machine is a builder. if slave_type == 'Builder': factory_cmd_obj.AddZipBuild(halt_on_failure=True, factory_properties=factory_properties) # Download the full output directory if the machine is a tester. if slave_type == 'Tester': factory_cmd_obj.AddExtractBuild(build_url, factory_properties=factory_properties) return factory
def AddUpdateStep(self, gclient_spec, factory_properties, factory, slave_type, sudo_for_remove=False, gclient_deps=None): if gclient_spec is None: gclient_spec = self.BuildGClientSpec() factory_properties = factory_properties or {} # Get the factory command object to add update step to the factory. factory_cmd_obj = commands.FactoryCommands(factory, target_platform=self._target_platform) # Get variables needed for the update. env = factory_properties.get('gclient_env', {}) timeout = factory_properties.get('gclient_timeout') no_gclient_branch = factory_properties.get('no_gclient_branch', False) gclient_transitive = factory_properties.get('gclient_transitive', False) primary_repo = factory_properties.get('primary_repo', '') gclient_jobs = factory_properties.get('gclient_jobs') # Add the update step. factory_cmd_obj.AddUpdateStep( gclient_spec, env=env, timeout=timeout, sudo_for_remove=sudo_for_remove, gclient_deps=gclient_deps, gclient_nohooks=True, no_gclient_branch=no_gclient_branch, gclient_transitive=gclient_transitive, primary_repo=primary_repo, gclient_jobs=gclient_jobs) if slave_type == 'Trybot': factory_cmd_obj.AddApplyIssueStep() if not self._nohooks_on_update: factory_cmd_obj.AddRunHooksStep(env=env, timeout=timeout)
def BaseFactory(self, gclient_spec=None, official_release=False, factory_properties=None, build_properties=None, delay_compile_step=False, sudo_for_remove=False, gclient_deps=None, slave_type=None): if gclient_spec is None: gclient_spec = self.BuildGClientSpec() factory_properties = factory_properties or {} factory = BuildFactory(build_properties) factory_cmd_obj = commands.FactoryCommands(factory, target_platform=self._target_platform) # First kill any svn.exe tasks so we can update in peace, and # afterwards use the checked-out script to kill everything else. if (self._target_platform == 'win32' and not factory_properties.get('no_kill')): factory_cmd_obj.AddSvnKillStep() factory_cmd_obj.AddUpdateScriptStep( gclient_jobs=factory_properties.get('update_scripts_gclient_jobs')) # Once the script is updated, the zombie processes left by the previous # run can be killed. if (self._target_platform == 'win32' and not factory_properties.get('no_kill')): factory_cmd_obj.AddTaskkillStep() env = factory_properties.get('gclient_env', {}) # Allow gclient_deps to also come from the factory_properties. if gclient_deps == None: gclient_deps = factory_properties.get('gclient_deps', None) # svn timeout is 2 min; we allow 5 timeout = factory_properties.get('gclient_timeout') if official_release or factory_properties.get('nuke_and_pave'): no_gclient_branch = factory_properties.get('no_gclient_branch', False) factory_cmd_obj.AddClobberTreeStep(gclient_spec, env, timeout, gclient_deps=gclient_deps, gclient_nohooks=self._nohooks_on_update, no_gclient_branch=no_gclient_branch) elif not delay_compile_step: self.AddUpdateStep(gclient_spec, factory_properties, factory, slave_type, sudo_for_remove, gclient_deps=gclient_deps) return factory
def BuildFactory(self, target='Release', clobber=False, tests=None, mode=None, slave_type='BuilderTester', options=None, compile_timeout=1200, build_url=None, project=None, factory_properties=None, gclient_deps=None, target_arch=None, skip_archive_steps=False): if factory_properties is None: factory_properties = {} factory_properties.setdefault('gclient_env', {}) gclient_env = factory_properties['gclient_env'] # Create the spec for the solutions gclient_spec = self.BuildGClientSpec(tests) # Initialize the factory with the basic steps. factory = self.BaseFactory(gclient_spec, factory_properties=factory_properties, slave_type=slave_type, gclient_deps=gclient_deps, options=options) # Optional repository root (default: 'src'). repository_root = factory_properties.get('repository_root', 'src') # Get the factory command object to create new steps to the factory. factory_cmd_obj = commands.FactoryCommands(factory, target, self._build_dir, self._target_platform, target_arch, repository_root) # Update clang if necessary. if ('clang=1' in gclient_env.get('GYP_DEFINES', '') or (self._target_platform != 'win32' and factory_properties.get('asan'))): factory_cmd_obj.AddUpdateClangStep() # Add a step to cleanup temporary files and data left from a previous run # to prevent the drives from becoming full over time. factory_cmd_obj.AddTempCleanupStep() # Update the NaCl SDK if needed if factory_properties.get('update_nacl_sdk'): factory_cmd_obj.AddUpdateNaClSDKStep( factory_properties['update_nacl_sdk']) # Add the compile step if needed. if slave_type in [ 'BuilderTester', 'Builder', 'Trybot', 'Indexer', 'TrybotBuilder' ]: # If we want to confirm that two successive compiles result in a no-op # build, tell the compile step to verify that. if factory_properties.get('confirm_noop_compile'): options = ['--ninja-ensure-up-to-date'] + options factory_cmd_obj.AddCompileStep( project, clobber, mode=mode, options=options, timeout=compile_timeout, env=factory_properties.get('compile_env')) # Generate synthetic user profiles. Must run before AddZipBuild(). if factory_properties.get('create_profiles'): # pylint: disable=W0212 factory_cmd_obj.AddProfileCreationStep('small_profile') if not skip_archive_steps: # Archive the full output directory if the machine is a builder. if slave_type in ['Builder', 'TrybotBuilder']: if build_url or factory_properties.get('build_url'): # There are some builders that are classified as builders, but only # because we only need to see if the patch compiles, and not # actually because triggered testers needs the build. So if we don't # supply a build_url, we assume that the build artifact doesn't # need to be uploaded and we can bypass this step. factory_cmd_obj.AddZipBuild( build_url, factory_properties=factory_properties) # Download the full output directory if the machine is a tester. if slave_type in ['Tester', 'TrybotTester']: factory_cmd_obj.AddExtractBuild() return factory
def BaseFactory(self, gclient_spec=None, official_release=False, factory_properties=None, build_properties=None, sudo_for_remove=False, gclient_deps=None, slave_type=None, options=None, target=None): if gclient_spec is None: gclient_spec = self.BuildGClientSpec() factory_properties = factory_properties or {} factory = BuildFactory(build_properties) factory_cmd_obj = commands.FactoryCommands( factory, target_platform=self._target_platform) # First kill any svn.exe tasks so we can update in peace, and # afterwards use the checked-out script to kill everything else. if (self._target_platform == 'win32' and not factory_properties.get('no_kill')): factory_cmd_obj.AddSvnKillStep() extra_args = None if factory_properties.get('goma_canary'): extra_args = ['--revision', 'build/goma@HEAD'] factory_cmd_obj.AddUpdateScriptStep( gclient_jobs=factory_properties.get('update_scripts_gclient_jobs'), args=extra_args) # Once the script is updated, the zombie processes left by the previous # run can be killed. if (self._target_platform == 'win32' and not factory_properties.get('no_kill')): factory_cmd_obj.AddTaskkillStep() env = factory_properties.get('gclient_env', {}) # Allow gclient_deps to also come from the factory_properties. if gclient_deps == None: gclient_deps = factory_properties.get('gclient_deps', None) if gclient_deps == 'ios': gclient_spec += ';target_os = [\'ios\'];target_os_only = True' # Do we need to operate in magic blink mode? blink_config = factory_properties.get('blink_config') # Force the build checkout to be at some revision. This may or may not # activate depending on its own criteria, but the expectation is that if # this does activate, it will emit a BOT_UPDATED file in the build/ # directory to signal to the other gclient update steps to no-op. code_review_site = config.Master.Master4.code_review_site factory_cmd_obj.AddBotUpdateStep(env, gclient_spec, self._revision_mapping, server=code_review_site, blink_config=blink_config) use_mb = factory_properties.get('use_mb') if use_mb: # Don't run GYP as part of gclient runhooks. env.update({'GYP_CHROMIUM_NO_ACTION': '1'}) # svn timeout is 2 min; we allow 5 timeout = factory_properties.get('gclient_timeout') if official_release or factory_properties.get('nuke_and_pave'): no_gclient_branch = factory_properties.get('no_gclient_branch', False) factory_cmd_obj.AddClobberTreeStep( gclient_spec, env, timeout, gclient_deps=gclient_deps, gclient_nohooks=self._nohooks_on_update, no_gclient_branch=no_gclient_branch, options=options) else: # Revert the tree to a clean (unmodified) state. factory_cmd_obj.AddGClientRevertStep() self.AddUpdateStep(gclient_spec, factory_properties, factory, slave_type, sudo_for_remove, gclient_deps=gclient_deps, options=options, blink_config=blink_config) if use_mb: # To be safe, we reset env back to its non-MB state and then make sure # MB is passed an env w/ no GYP flags set at all. del env['GYP_CHROMIUM_NO_ACTION'] mb_env = env.copy() if 'GYP_DEFINES' in mb_env: del mb_env['GYP_DEFINES'] if 'GYP_CROSSCOMPILE' in mb_env: del mb_env['GYP_CROSSCOMPILE'] factory_cmd_obj.AddGenerateBuildFilesStep(env=mb_env, timeout=timeout, options=options, target=target) return factory
def BuildFactory(self, target='Release', clobber=False, tests=None, mode=None, slave_type='BuilderTester', options=None, compile_timeout=1200, build_url=None, project=None, factory_properties=None, gclient_deps=None, target_arch=None, skip_archive_steps=False): if factory_properties is None: factory_properties = {} factory_properties.setdefault('gclient_env', {}) gclient_env = factory_properties['gclient_env'] if options and '--build-tool=ninja' in options: gclient_env['GYP_GENERATORS'] = 'ninja' # Create the spec for the solutions gclient_spec = self.BuildGClientSpec(tests) # Initialize the factory with the basic steps. factory = self.BaseFactory(gclient_spec, factory_properties=factory_properties, slave_type=slave_type, gclient_deps=gclient_deps, options=options) # Optional repository root (default: 'src'). repository_root = factory_properties.get('repository_root', 'src') # Get the factory command object to create new steps to the factory. factory_cmd_obj = commands.FactoryCommands(factory, target, self._build_dir, self._target_platform, target_arch, repository_root) # Update clang if necessary. if ('clang=1' in gclient_env.get('GYP_DEFINES', '') or (self._target_platform != 'win32' and factory_properties.get('asan'))): factory_cmd_obj.AddUpdateClangStep() # Add a step to cleanup temporary files and data left from a previous run # to prevent the drives from becoming full over time. factory_cmd_obj.AddTempCleanupStep() # Update the NaCl SDK if needed if factory_properties.get('update_nacl_sdk'): factory_cmd_obj.AddUpdateNaClSDKStep( factory_properties['update_nacl_sdk']) # Add the compile step if needed. if slave_type in [ 'BuilderTester', 'Builder', 'Trybot', 'Indexer', 'TrybotBuilder' ]: if self._target_platform == 'win32': # Look for a solution named for its enclosing directory. project = project or os.path.basename(self._build_dir) + '.sln' factory_cmd_obj.AddCompileStep( project, clobber, mode=mode, options=options, timeout=compile_timeout, env=factory_properties.get('compile_env')) # Generate synthetic user profiles. Must run before AddZipBuild(). if factory_properties.get('create_profiles'): # pylint: disable=W0212 factory_cmd_obj.AddProfileCreationStep('small_profile') if not skip_archive_steps: # Archive the full output directory if the machine is a builder. if slave_type in ['Builder', 'TrybotBuilder']: factory_cmd_obj.AddZipBuild( halt_on_failure=True, factory_properties=factory_properties) # Download the full output directory if the machine is a tester. if slave_type in ['Tester', 'TrybotTester']: factory_cmd_obj.AddExtractBuild( build_url, factory_properties=factory_properties) return factory
def BaseFactory(self, gclient_spec=None, official_release=False, factory_properties=None, build_properties=None, delay_compile_step=False, sudo_for_remove=False, gclient_deps=None, slave_type=None, options=None): if gclient_spec is None: gclient_spec = self.BuildGClientSpec() factory_properties = factory_properties or {} factory = BuildFactory(build_properties) factory_cmd_obj = commands.FactoryCommands( factory, target_platform=self._target_platform) # First kill any svn.exe tasks so we can update in peace, and # afterwards use the checked-out script to kill everything else. if (self._target_platform == 'win32' and not factory_properties.get('no_kill')): factory_cmd_obj.AddSvnKillStep() script_solutions = None if (factory_properties.get('goma_canary') and factory_properties.get('slave_internal_url')): script_solutions = [ GClientSolution(factory_properties.get('slave_internal_url'), name="slave.DEPS", custom_vars_list=[ ("goma_linux_revision", "HEAD"), ("goma_mac_revision", "HEAD"), ("goma_win_revision", "HEAD") ]) ] factory_cmd_obj.AddUpdateScriptStep( gclient_jobs=factory_properties.get('update_scripts_gclient_jobs'), solutions=script_solutions) # Once the script is updated, the zombie processes left by the previous # run can be killed. if (self._target_platform == 'win32' and not factory_properties.get('no_kill')): factory_cmd_obj.AddTaskkillStep() env = factory_properties.get('gclient_env', {}) # Allow gclient_deps to also come from the factory_properties. if gclient_deps == None: gclient_deps = factory_properties.get('gclient_deps', None) if gclient_deps == 'ios': gclient_spec += ';target_os = [\'ios\'];target_os_only = True' # Force the build checkout to be at some revision. This may or may not # activate depending on its own criteria, but the expectation is that if # this does activate, it will emit a BOT_UPDATED file in the build/ # directory to signal to the other gclient update steps to no-op. code_review_site = config.Master.Master4.code_review_site factory_cmd_obj.AddBotUpdateStep(env, gclient_spec, self._revision_mapping, server=code_review_site) # svn timeout is 2 min; we allow 5 timeout = factory_properties.get('gclient_timeout') if official_release or factory_properties.get('nuke_and_pave'): no_gclient_branch = factory_properties.get('no_gclient_branch', False) factory_cmd_obj.AddClobberTreeStep( gclient_spec, env, timeout, gclient_deps=gclient_deps, gclient_nohooks=self._nohooks_on_update, no_gclient_branch=no_gclient_branch, options=options) else: # Revert the tree to a clean (unmodified) state. factory_cmd_obj.AddGClientRevertStep() if not delay_compile_step: self.AddUpdateStep(gclient_spec, factory_properties, factory, slave_type, sudo_for_remove, gclient_deps=gclient_deps, options=options) return factory