def setUp(self): os.makedirs(os.path.join(self.tempdir, '.repo')) self.source_repo = 'ssh://source/repo' self.manifest_repo = 'ssh://manifest/repo' self.version_file = 'version-file.sh' self.branch = 'master' self.build_names = ['x86-generic-paladin'] self.incr_type = 'branch' repo = repository.RepoRepository(self.source_repo, self.tempdir, self.branch) self.manager = manifest_version.BuildSpecsManager(repo, self.manifest_repo, self.build_names, self.incr_type, False, branch=self.branch, dry_run=True) # Change default to something we clean up. self.tmpmandir = os.path.join(self.tempdir, 'man') osutils.SafeMakedirs(self.tmpmandir) self.manager.manifest_dir = self.tmpmandir # Shorten the sleep between attempts. self.manager.SLEEP_TIMEOUT = 1
def BuildManager(self, config=None, metadata=None, db=None, buildbucket_client=None): db = db or self.db repo = repository.RepoRepository(self.source_repo, self.tempdir, self.branch) manager = manifest_version.BuildSpecsManager( repo, self.manifest_repo, self.build_names, self.incr_type, False, branch=self.branch, dry_run=True, config=config, metadata=metadata, db=db, buildbucket_client=buildbucket_client) manager.manifest_dir = self.tmpmandir # Shorten the sleep between attempts. manager.SLEEP_TIMEOUT = 1 return manager
def setUp(self): self.repo = repository.RepoRepository(site_config.params.MANIFEST_URL, self.tempdir, branch='master') self.PatchObject(repository.RepoRepository, 'Initialize') self.PatchObject(repository.RepoRepository, '_EnsureMirroring') self.PatchObject(repository.RepoRepository, 'BuildRootGitCleanup') self.PatchObject(time, 'sleep')
def main(argv): parser = GetParser() options = parser.parse_args(argv) options.Freeze() local_manifest = ResolveLocalManifestPath(options) if local_manifest: logging.info('Using local_manifest: %s', local_manifest) if options.manifest_url: manifest_url = options.manifest_url elif options.external: manifest_url = config_lib.GetSiteParams().MANIFEST_URL else: manifest_url = config_lib.GetSiteParams().MANIFEST_INT_URL osutils.SafeMakedirs(options.repo_root) repo = repository.RepoRepository( manifest_repo_url=manifest_url, directory=options.repo_root, branch=options.branch, git_cache_dir=options.git_cache_dir, repo_url=options.repo_url, groups=options.groups) if options.copy_repo: repo.PreLoad(options.copy_repo) if repository.IsARepoRoot(options.repo_root): repo.BuildRootGitCleanup(prune_all=True) repo.Sync(local_manifest=local_manifest, detach=True) if options.gerrit_patches: patches = gerrit.GetGerritPatchInfo(options.gerrit_patches) # TODO: Extract patches from manifest synced. helper_pool = patch_series.HelperPool.SimpleCreate( cros_internal=not options.external, cros=True) series = patch_series.PatchSeries( path=options.repo_root, helper_pool=helper_pool, forced_manifest=None) _, failed_tot, failed_inflight = series.Apply(patches) failed = failed_tot + failed_inflight if failed: logging.error('Failed to apply: %s', ', '.join(str(p) for p in failed)) return 1
def GetRepoRepository(self, **kwargs): """Create a new repo repository object.""" manifest_url = self._run.options.manifest_repo_url if manifest_url is None: manifest_url = self._run.config.manifest_repo_url manifest_branch = self._run.config.manifest_branch if manifest_branch is None: manifest_branch = self._run.manifest_branch kwargs.setdefault('referenced_repo', self._run.options.reference_repo) kwargs.setdefault('branch', manifest_branch) kwargs.setdefault('manifest', self._run.config.manifest) kwargs.setdefault('git_cache_dir', self._run.options.git_cache_dir) # pass in preserve_paths so that repository.RepoRepository # knows what paths to preserve when executing clean_up_repo if hasattr(self._run.options, 'preserve_paths'): kwargs.setdefault('preserve_paths', self._run.options.preserve_paths) return repository.RepoRepository(manifest_url, self._build_root, **kwargs)
def setUp(self): self.push_mock = self.PatchObject(git, 'CreatePushBranch') self.source_repo = 'ssh://source/repo' self.manifest_repo = 'ssh://manifest/repo' self.version_file = 'version-file.sh' self.branch = 'master' self.build_name = 'amd64-generic' self.incr_type = 'branch' self.buildstore = FakeBuildStore() # Create tmp subdirs based on the one provided TempDirMixin. self.tmpdir = os.path.join(self.tempdir, 'base') osutils.SafeMakedirs(self.tmpdir) self.tmpmandir = os.path.join(self.tempdir, 'man') osutils.SafeMakedirs(self.tmpmandir) repo = repository.RepoRepository(self.source_repo, self.tmpdir, self.branch, depth=1) self.manager = lkgm_manager.LKGMManager(repo, self.manifest_repo, self.build_name, constants.PFQ_TYPE, 'branch', force=False, branch=self.branch, buildstore=self.buildstore, dry_run=True) self.manager.manifest_dir = self.tmpmandir self.manager.lkgm_path = os.path.join(self.tmpmandir, constants.LKGM_MANIFEST) self.manager.all_specs_dir = '/LKGM/path' manifest_dir = self.manager.manifest_dir self.manager.specs_for_builder = os.path.join( manifest_dir, self.manager.rel_working_dir, 'build-name', '%(builder)s') self.manager.SLEEP_TIMEOUT = 0
def setUp(self): self.manifest_dir = os.path.join(self.tempdir, '.repo', 'manifests') # Initialize a repo instance here. local_repo = os.path.join(constants.SOURCE_ROOT, '.repo/repo/.git') # TODO(evanhernandez): This is a hack. Find a way to simplify this test. # We used to use the current checkout's manifests.git, but that caused # problems in production environemnts. remote_manifests = os.path.join(self.tempdir, 'remote', 'manifests.git') osutils.SafeMakedirs(remote_manifests) git.Init(remote_manifests) default_manifest = os.path.join(remote_manifests, 'default.xml') osutils.WriteFile( default_manifest, '<?xml version="1.0" encoding="UTF-8"?><manifest></manifest>') git.AddPath(default_manifest) git.Commit(remote_manifests, 'dummy commit', allow_empty=True) git.CreateBranch(remote_manifests, 'default') git.CreateBranch(remote_manifests, 'release-R23-2913.B') git.CreateBranch(remote_manifests, 'release-R23-2913.B-suffix') git.CreateBranch(remote_manifests, 'firmware-link-') # Create a copy of our existing manifests.git, but rewrite it so it # looks like a remote manifests.git. This is to avoid hitting the # network, and speeds things up in general. local_manifests = 'file://%s' % remote_manifests temp_manifests = os.path.join(self.tempdir, 'manifests.git') git.RunGit(self.tempdir, ['clone', '-n', '--bare', local_manifests]) git.RunGit(temp_manifests, ['fetch', '-f', '-u', local_manifests, 'refs/remotes/origin/*:refs/heads/*']) git.RunGit(temp_manifests, ['branch', '-D', 'default']) repo = repository.RepoRepository( temp_manifests, self.tempdir, repo_url='file://%s' % local_repo, repo_branch='default') repo.Initialize() self.active_manifest = os.path.realpath( os.path.join(self.tempdir, '.repo', 'manifest.xml'))
def setUp(self): self.source_repo = 'ssh://source/repo' self.manifest_version_url = 'fake manifest url' self.branch = 'master' self.build_name = 'amd64-generic' self.incr_type = 'branch' self.next_version = 'next_version' self.sync_stage = None self.buildstore = FakeBuildStore() self.repo = repository.RepoRepository(self.source_repo, self.tempdir, self.branch) self.manager = manifest_version.BuildSpecsManager( self.repo, self.manifest_version_url, [self.build_name], self.incr_type, force=False, branch=self.branch, buildstore=self.buildstore, dry_run=True) self._Prepare()
def setUp(self): """Setup""" self.source_repo = 'ssh://source/repo' self.manifest_version_url = 'fake manifest url' self.branch = 'master' self.build_name = 'master-mst-android-pfq' self.incr_type = 'branch' self.next_version = 'next_version' self.sync_stage = None self.repo = repository.RepoRepository(self.source_repo, self.tempdir, self.branch) # Create and set up a fake cidb instance. self.fake_db = fake_cidb.FakeCIDBConnection() self.buildstore = FakeBuildStore(self.fake_db) cidb.CIDBConnectionFactory.SetupMockCidb(self.fake_db) self.manager = lkgm_manager.LKGMManager( source_repo=self.repo, manifest_repo=self.manifest_version_url, build_names=[self.build_name], build_type=constants.ANDROID_PFQ_TYPE, incr_type=self.incr_type, force=False, branch=self.branch, buildstore=self.buildstore, dry_run=True) self.PatchObject(buildbucket_lib, 'GetServiceAccount', return_value=True) self.PatchObject(auth.AuthorizedHttp, '__init__', return_value=None) self.PatchObject(buildbucket_lib.BuildbucketClient, '_GetHost', return_value=buildbucket_lib.BUILDBUCKET_TEST_HOST) self._Prepare()
def _Initialize(self, branch='master'): repo = repository.RepoRepository(site_config.params.MANIFEST_URL, self.tempdir, branch=branch) repo.Initialize()
def _main(options, argv): """main method of script. Args: options: preparsed options object for the build. argv: All command line arguments to pass as list of strings. Returns: Return code of cbuildbot as an integer. """ branchname = options.branch or 'master' root = options.buildroot buildroot = os.path.join(root, 'repository') workspace = os.path.join(root, 'workspace') depot_tools_path = os.path.join(buildroot, constants.DEPOT_TOOLS_SUBPATH) # Does the entire build pass or fail. with metrics.Presence(METRIC_ACTIVE), \ metrics.SuccessCounter(METRIC_COMPLETED) as s_fields: # Preliminary set, mostly command line parsing. with metrics.SuccessCounter(METRIC_INVOKED): if options.enable_buildbot_tags: logging.EnableBuildbotMarkers() ConfigureGlobalEnvironment() # Prepare the buildroot with source for the build. with metrics.SuccessCounter(METRIC_PREP): manifest_url = config_lib.GetSiteParams().MANIFEST_INT_URL repo = repository.RepoRepository( manifest_url, buildroot, branch=branchname, git_cache_dir=options.git_cache_dir) previous_build_state = GetLastBuildState(root) # Clean up the buildroot to a safe state. with metrics.SecondsTimer(METRIC_CLEAN): build_state = GetCurrentBuildState(options, branchname) CleanBuildRoot(root, repo, options.cache_dir, build_state) # Get a checkout close enough to the branch that cbuildbot can handle it. if options.sync: with metrics.SecondsTimer(METRIC_INITIAL): InitialCheckout(repo) # Run cbuildbot inside the full ChromeOS checkout, on the specified branch. with metrics.SecondsTimer(METRIC_CBUILDBOT), \ metrics.SecondsInstanceTimer(METRIC_CBUILDBOT_INSTANCE): if previous_build_state.is_valid(): argv.append('--previous-build-state') argv.append( base64.b64encode(previous_build_state.to_json().encode( 'utf-8')).decode('utf-8')) argv.extend(['--workspace', workspace]) if not options.cache_dir_specified: argv.extend(['--cache-dir', options.cache_dir]) result = Cbuildbot(buildroot, depot_tools_path, argv) s_fields['success'] = (result == 0) build_state.status = (constants.BUILDER_STATUS_PASSED if result == 0 else constants.BUILDER_STATUS_FAILED) SetLastBuildState(root, build_state) with metrics.SecondsTimer(METRIC_CHROOT_CLEANUP): CleanupChroot(buildroot) return result
def _Initialize(self, branch='master'): site_params = config_lib.GetSiteParams() self.repo = repository.RepoRepository(site_params.MANIFEST_URL, self.tempdir, branch=branch) self.repo.Initialize()
def _main(argv): """main method of script. Args: argv: All command line arguments to pass as list of strings. Returns: Return code of cbuildbot as an integer. """ options = PreParseArguments(argv) branchname = options.branch or 'master' root = options.buildroot buildroot = os.path.join(root, 'repository') depot_tools_path = os.path.join(buildroot, constants.DEPOT_TOOLS_SUBPATH) metrics_fields = { 'branch_name': branchname, 'build_config': options.build_config_name, 'tryjob': options.remote_trybot, } # Does the entire build pass or fail. with metrics.Presence(METRIC_ACTIVE, metrics_fields), \ metrics.SuccessCounter(METRIC_COMPLETED, metrics_fields) as s_fields: # Preliminary set, mostly command line parsing. with metrics.SuccessCounter(METRIC_INVOKED, metrics_fields): if options.enable_buildbot_tags: logging.EnableBuildbotMarkers() ConfigureGlobalEnvironment() # Prepare the buildroot with source for the build. with metrics.SuccessCounter(METRIC_PREP, metrics_fields): site_config = config_lib.GetConfig() manifest_url = site_config.params['MANIFEST_INT_URL'] repo = repository.RepoRepository( manifest_url, buildroot, branch=branchname, git_cache_dir=options.git_cache_dir) previous_build_state = GetLastBuildState(root) # Clean up the buildroot to a safe state. with metrics.SecondsTimer(METRIC_CLEAN, fields=metrics_fields): build_state = GetCurrentBuildState(options, branchname) CleanBuildRoot(root, repo, metrics_fields, build_state) # Get a checkout close enough to the branch that cbuildbot can handle it. if options.sync: with metrics.SecondsTimer(METRIC_INITIAL, fields=metrics_fields): InitialCheckout(repo) # Get a checkout close enough to the branch that cbuildbot can handle it. with metrics.SecondsTimer(METRIC_DEPOT_TOOLS, fields=metrics_fields): DepotToolsEnsureBootstrap(depot_tools_path) # Run cbuildbot inside the full ChromeOS checkout, on the specified branch. with metrics.SecondsTimer(METRIC_CBUILDBOT, fields=metrics_fields): if previous_build_state.is_valid(): argv.append('--previous-build-state') argv.append(base64.b64encode(previous_build_state.to_json())) result = Cbuildbot(buildroot, depot_tools_path, argv) s_fields['success'] = (result == 0) build_state.status = (constants.BUILDER_STATUS_PASSED if result == 0 else constants.BUILDER_STATUS_FAILED) SetLastBuildState(root, build_state) CleanupChroot(buildroot) return result