def setUp(self):
os.makedirs(os.path.join(self.tempdir, '.repo'))
self.source_repo = 'ssh://source/repo'
self.manifest_repo = 'ssh://manifest/repo'
self.version_file = 'version-file.sh'
self.branch = 'master'
self.build_names = ['x86-generic-paladin']
self.incr_type = 'branch'
repo = repository.RepoRepository(self.source_repo, self.tempdir,
self.branch)
self.manager = manifest_version.BuildSpecsManager(repo,
self.manifest_repo,
self.build_names,
self.incr_type,
False,
branch=self.branch,
dry_run=True)
# Change default to something we clean up.
self.tmpmandir = os.path.join(self.tempdir, 'man')
osutils.SafeMakedirs(self.tmpmandir)
self.manager.manifest_dir = self.tmpmandir
# Shorten the sleep between attempts.
self.manager.SLEEP_TIMEOUT = 1
def BuildManager(self,
config=None,
metadata=None,
db=None,
buildbucket_client=None):
db = db or self.db
repo = repository.RepoRepository(self.source_repo, self.tempdir,
self.branch)
manager = manifest_version.BuildSpecsManager(
repo,
self.manifest_repo,
self.build_names,
self.incr_type,
False,
branch=self.branch,
dry_run=True,
config=config,
metadata=metadata,
db=db,
buildbucket_client=buildbucket_client)
manager.manifest_dir = self.tmpmandir
# Shorten the sleep between attempts.
manager.SLEEP_TIMEOUT = 1
return manager
def setUp(self):
self.repo = repository.RepoRepository(site_config.params.MANIFEST_URL,
self.tempdir,
branch='master')
self.PatchObject(repository.RepoRepository, 'Initialize')
self.PatchObject(repository.RepoRepository, '_EnsureMirroring')
self.PatchObject(repository.RepoRepository, 'BuildRootGitCleanup')
self.PatchObject(time, 'sleep')
def main(argv):
parser = GetParser()
options = parser.parse_args(argv)
options.Freeze()
local_manifest = ResolveLocalManifestPath(options)
if local_manifest:
logging.info('Using local_manifest: %s', local_manifest)
if options.manifest_url:
manifest_url = options.manifest_url
elif options.external:
manifest_url = config_lib.GetSiteParams().MANIFEST_URL
else:
manifest_url = config_lib.GetSiteParams().MANIFEST_INT_URL
osutils.SafeMakedirs(options.repo_root)
repo = repository.RepoRepository(
manifest_repo_url=manifest_url,
directory=options.repo_root,
branch=options.branch,
git_cache_dir=options.git_cache_dir,
repo_url=options.repo_url,
groups=options.groups)
if options.copy_repo:
repo.PreLoad(options.copy_repo)
if repository.IsARepoRoot(options.repo_root):
repo.BuildRootGitCleanup(prune_all=True)
repo.Sync(local_manifest=local_manifest, detach=True)
if options.gerrit_patches:
patches = gerrit.GetGerritPatchInfo(options.gerrit_patches)
# TODO: Extract patches from manifest synced.
helper_pool = patch_series.HelperPool.SimpleCreate(
cros_internal=not options.external, cros=True)
series = patch_series.PatchSeries(
path=options.repo_root, helper_pool=helper_pool, forced_manifest=None)
_, failed_tot, failed_inflight = series.Apply(patches)
failed = failed_tot + failed_inflight
if failed:
logging.error('Failed to apply: %s', ', '.join(str(p) for p in failed))
return 1
def GetRepoRepository(self, **kwargs):
"""Create a new repo repository object."""
manifest_url = self._run.options.manifest_repo_url
if manifest_url is None:
manifest_url = self._run.config.manifest_repo_url
manifest_branch = self._run.config.manifest_branch
if manifest_branch is None:
manifest_branch = self._run.manifest_branch
kwargs.setdefault('referenced_repo', self._run.options.reference_repo)
kwargs.setdefault('branch', manifest_branch)
kwargs.setdefault('manifest', self._run.config.manifest)
kwargs.setdefault('git_cache_dir', self._run.options.git_cache_dir)
# pass in preserve_paths so that repository.RepoRepository
# knows what paths to preserve when executing clean_up_repo
if hasattr(self._run.options, 'preserve_paths'):
kwargs.setdefault('preserve_paths', self._run.options.preserve_paths)
return repository.RepoRepository(manifest_url, self._build_root, **kwargs)
def setUp(self):
self.push_mock = self.PatchObject(git, 'CreatePushBranch')
self.source_repo = 'ssh://source/repo'
self.manifest_repo = 'ssh://manifest/repo'
self.version_file = 'version-file.sh'
self.branch = 'master'
self.build_name = 'amd64-generic'
self.incr_type = 'branch'
self.buildstore = FakeBuildStore()
# Create tmp subdirs based on the one provided TempDirMixin.
self.tmpdir = os.path.join(self.tempdir, 'base')
osutils.SafeMakedirs(self.tmpdir)
self.tmpmandir = os.path.join(self.tempdir, 'man')
osutils.SafeMakedirs(self.tmpmandir)
repo = repository.RepoRepository(self.source_repo,
self.tmpdir,
self.branch,
depth=1)
self.manager = lkgm_manager.LKGMManager(repo,
self.manifest_repo,
self.build_name,
constants.PFQ_TYPE,
'branch',
force=False,
branch=self.branch,
buildstore=self.buildstore,
dry_run=True)
self.manager.manifest_dir = self.tmpmandir
self.manager.lkgm_path = os.path.join(self.tmpmandir,
constants.LKGM_MANIFEST)
self.manager.all_specs_dir = '/LKGM/path'
manifest_dir = self.manager.manifest_dir
self.manager.specs_for_builder = os.path.join(
manifest_dir, self.manager.rel_working_dir, 'build-name',
'%(builder)s')
self.manager.SLEEP_TIMEOUT = 0
def setUp(self):
self.manifest_dir = os.path.join(self.tempdir, '.repo', 'manifests')
# Initialize a repo instance here.
local_repo = os.path.join(constants.SOURCE_ROOT, '.repo/repo/.git')
# TODO(evanhernandez): This is a hack. Find a way to simplify this test.
# We used to use the current checkout's manifests.git, but that caused
# problems in production environemnts.
remote_manifests = os.path.join(self.tempdir, 'remote', 'manifests.git')
osutils.SafeMakedirs(remote_manifests)
git.Init(remote_manifests)
default_manifest = os.path.join(remote_manifests, 'default.xml')
osutils.WriteFile(
default_manifest,
'<?xml version="1.0" encoding="UTF-8"?><manifest></manifest>')
git.AddPath(default_manifest)
git.Commit(remote_manifests, 'dummy commit', allow_empty=True)
git.CreateBranch(remote_manifests, 'default')
git.CreateBranch(remote_manifests, 'release-R23-2913.B')
git.CreateBranch(remote_manifests, 'release-R23-2913.B-suffix')
git.CreateBranch(remote_manifests, 'firmware-link-')
# Create a copy of our existing manifests.git, but rewrite it so it
# looks like a remote manifests.git. This is to avoid hitting the
# network, and speeds things up in general.
local_manifests = 'file://%s' % remote_manifests
temp_manifests = os.path.join(self.tempdir, 'manifests.git')
git.RunGit(self.tempdir, ['clone', '-n', '--bare', local_manifests])
git.RunGit(temp_manifests,
['fetch', '-f', '-u', local_manifests,
'refs/remotes/origin/*:refs/heads/*'])
git.RunGit(temp_manifests, ['branch', '-D', 'default'])
repo = repository.RepoRepository(
temp_manifests, self.tempdir,
repo_url='file://%s' % local_repo, repo_branch='default')
repo.Initialize()
self.active_manifest = os.path.realpath(
os.path.join(self.tempdir, '.repo', 'manifest.xml'))
def setUp(self):
self.source_repo = 'ssh://source/repo'
self.manifest_version_url = 'fake manifest url'
self.branch = 'master'
self.build_name = 'amd64-generic'
self.incr_type = 'branch'
self.next_version = 'next_version'
self.sync_stage = None
self.buildstore = FakeBuildStore()
self.repo = repository.RepoRepository(self.source_repo, self.tempdir,
self.branch)
self.manager = manifest_version.BuildSpecsManager(
self.repo,
self.manifest_version_url, [self.build_name],
self.incr_type,
force=False,
branch=self.branch,
buildstore=self.buildstore,
dry_run=True)
self._Prepare()
def setUp(self):
"""Setup"""
self.source_repo = 'ssh://source/repo'
self.manifest_version_url = 'fake manifest url'
self.branch = 'master'
self.build_name = 'master-mst-android-pfq'
self.incr_type = 'branch'
self.next_version = 'next_version'
self.sync_stage = None
self.repo = repository.RepoRepository(self.source_repo, self.tempdir,
self.branch)
# Create and set up a fake cidb instance.
self.fake_db = fake_cidb.FakeCIDBConnection()
self.buildstore = FakeBuildStore(self.fake_db)
cidb.CIDBConnectionFactory.SetupMockCidb(self.fake_db)
self.manager = lkgm_manager.LKGMManager(
source_repo=self.repo,
manifest_repo=self.manifest_version_url,
build_names=[self.build_name],
build_type=constants.ANDROID_PFQ_TYPE,
incr_type=self.incr_type,
force=False,
branch=self.branch,
buildstore=self.buildstore,
dry_run=True)
self.PatchObject(buildbucket_lib,
'GetServiceAccount',
return_value=True)
self.PatchObject(auth.AuthorizedHttp, '__init__', return_value=None)
self.PatchObject(buildbucket_lib.BuildbucketClient,
'_GetHost',
return_value=buildbucket_lib.BUILDBUCKET_TEST_HOST)
self._Prepare()
def _Initialize(self, branch='master'):
repo = repository.RepoRepository(site_config.params.MANIFEST_URL,
self.tempdir,
branch=branch)
repo.Initialize()
def _main(options, argv):
"""main method of script.
Args:
options: preparsed options object for the build.
argv: All command line arguments to pass as list of strings.
Returns:
Return code of cbuildbot as an integer.
"""
branchname = options.branch or 'master'
root = options.buildroot
buildroot = os.path.join(root, 'repository')
workspace = os.path.join(root, 'workspace')
depot_tools_path = os.path.join(buildroot, constants.DEPOT_TOOLS_SUBPATH)
# Does the entire build pass or fail.
with metrics.Presence(METRIC_ACTIVE), \
metrics.SuccessCounter(METRIC_COMPLETED) as s_fields:
# Preliminary set, mostly command line parsing.
with metrics.SuccessCounter(METRIC_INVOKED):
if options.enable_buildbot_tags:
logging.EnableBuildbotMarkers()
ConfigureGlobalEnvironment()
# Prepare the buildroot with source for the build.
with metrics.SuccessCounter(METRIC_PREP):
manifest_url = config_lib.GetSiteParams().MANIFEST_INT_URL
repo = repository.RepoRepository(
manifest_url,
buildroot,
branch=branchname,
git_cache_dir=options.git_cache_dir)
previous_build_state = GetLastBuildState(root)
# Clean up the buildroot to a safe state.
with metrics.SecondsTimer(METRIC_CLEAN):
build_state = GetCurrentBuildState(options, branchname)
CleanBuildRoot(root, repo, options.cache_dir, build_state)
# Get a checkout close enough to the branch that cbuildbot can handle it.
if options.sync:
with metrics.SecondsTimer(METRIC_INITIAL):
InitialCheckout(repo)
# Run cbuildbot inside the full ChromeOS checkout, on the specified branch.
with metrics.SecondsTimer(METRIC_CBUILDBOT), \
metrics.SecondsInstanceTimer(METRIC_CBUILDBOT_INSTANCE):
if previous_build_state.is_valid():
argv.append('--previous-build-state')
argv.append(
base64.b64encode(previous_build_state.to_json().encode(
'utf-8')).decode('utf-8'))
argv.extend(['--workspace', workspace])
if not options.cache_dir_specified:
argv.extend(['--cache-dir', options.cache_dir])
result = Cbuildbot(buildroot, depot_tools_path, argv)
s_fields['success'] = (result == 0)
build_state.status = (constants.BUILDER_STATUS_PASSED if result
== 0 else constants.BUILDER_STATUS_FAILED)
SetLastBuildState(root, build_state)
with metrics.SecondsTimer(METRIC_CHROOT_CLEANUP):
CleanupChroot(buildroot)
return result
def _Initialize(self, branch='master'):
site_params = config_lib.GetSiteParams()
self.repo = repository.RepoRepository(site_params.MANIFEST_URL,
self.tempdir,
branch=branch)
self.repo.Initialize()
def _main(argv):
"""main method of script.
Args:
argv: All command line arguments to pass as list of strings.
Returns:
Return code of cbuildbot as an integer.
"""
options = PreParseArguments(argv)
branchname = options.branch or 'master'
root = options.buildroot
buildroot = os.path.join(root, 'repository')
depot_tools_path = os.path.join(buildroot, constants.DEPOT_TOOLS_SUBPATH)
metrics_fields = {
'branch_name': branchname,
'build_config': options.build_config_name,
'tryjob': options.remote_trybot,
}
# Does the entire build pass or fail.
with metrics.Presence(METRIC_ACTIVE, metrics_fields), \
metrics.SuccessCounter(METRIC_COMPLETED, metrics_fields) as s_fields:
# Preliminary set, mostly command line parsing.
with metrics.SuccessCounter(METRIC_INVOKED, metrics_fields):
if options.enable_buildbot_tags:
logging.EnableBuildbotMarkers()
ConfigureGlobalEnvironment()
# Prepare the buildroot with source for the build.
with metrics.SuccessCounter(METRIC_PREP, metrics_fields):
site_config = config_lib.GetConfig()
manifest_url = site_config.params['MANIFEST_INT_URL']
repo = repository.RepoRepository(
manifest_url,
buildroot,
branch=branchname,
git_cache_dir=options.git_cache_dir)
previous_build_state = GetLastBuildState(root)
# Clean up the buildroot to a safe state.
with metrics.SecondsTimer(METRIC_CLEAN, fields=metrics_fields):
build_state = GetCurrentBuildState(options, branchname)
CleanBuildRoot(root, repo, metrics_fields, build_state)
# Get a checkout close enough to the branch that cbuildbot can handle it.
if options.sync:
with metrics.SecondsTimer(METRIC_INITIAL,
fields=metrics_fields):
InitialCheckout(repo)
# Get a checkout close enough to the branch that cbuildbot can handle it.
with metrics.SecondsTimer(METRIC_DEPOT_TOOLS,
fields=metrics_fields):
DepotToolsEnsureBootstrap(depot_tools_path)
# Run cbuildbot inside the full ChromeOS checkout, on the specified branch.
with metrics.SecondsTimer(METRIC_CBUILDBOT, fields=metrics_fields):
if previous_build_state.is_valid():
argv.append('--previous-build-state')
argv.append(base64.b64encode(previous_build_state.to_json()))
result = Cbuildbot(buildroot, depot_tools_path, argv)
s_fields['success'] = (result == 0)
build_state.status = (constants.BUILDER_STATUS_PASSED if result
== 0 else constants.BUILDER_STATUS_FAILED)
SetLastBuildState(root, build_state)
CleanupChroot(buildroot)
return result
