def test_clone_upstream(self):
git = self.git
test_parent = os.path.join(self.base_temp_dir, 'test_clone_upstream')
os.makedirs(test_parent)
test_dir = os.path.join(test_parent, TEST_REPO_NAME)
repository = GitRepositorySpec(TEST_REPO_NAME,
git_dir=test_dir,
origin=self.git_dir)
git.clone_repository_to_path(repository)
self.assertTrue(os.path.exists(os.path.join(test_dir, 'base_file')))
want_tags = git.query_tag_commits(self.git_dir, TAG_VERSION_PATTERN)
have_tags = git.query_tag_commits(test_dir, TAG_VERSION_PATTERN)
self.assertEqual(want_tags, have_tags)
got = check_subprocess('git -C "{dir}" remote -v'.format(dir=test_dir))
# Disable pushes to the origni
# No upstream since origin is upstream
self.assertEqual(
'\n'.join([
'origin\t{origin} (fetch)'.format(origin=self.git_dir),
'origin\tdisabled (push)'
]), got)
reference = git.determine_git_repository_spec(test_dir)
expect = GitRepositorySpec(os.path.basename(self.git_dir),
origin=self.git_dir,
git_dir=test_dir)
self.assertEqual(expect, reference)
def test_determine_most_common_prefix(self):
options = self.options
builder = BomBuilder(options, self.scm)
self.assertIsNone(builder.determine_most_common_prefix())
prefix = ['http://github.com/one', '/local/source/path/two']
# Test two vs one in from different repo prefix
# run the test twice changing the ordering the desired prefix is visible.
for which in [0, 1]:
repository = GitRepositorySpec(
'RepoOne', origin=prefix[0] + '/RepoOne',
commit_id='RepoOneCommit')
summary = RepositorySummary('RepoOneCommit', 'RepoOneTag',
'1.2.3', '1.2.2', [])
source_info = SourceInfo('BuildOne', summary)
builder.add_repository(repository, source_info)
self.assertEquals(prefix[0], builder.determine_most_common_prefix())
repository = GitRepositorySpec(
'RepoTwo', origin=prefix[which] + '/RepoTwo',
commit_id='RepoTwoCommit')
summary = RepositorySummary('RepoTwoCommit', 'RepoTwoTag',
'2.2.3', '2.2.3', [])
source_info = SourceInfo('BuildTwo', summary)
builder.add_repository(repository, source_info)
repository = GitRepositorySpec(
'RepoThree', origin=prefix[1] + '/RepoThree',
commit_id='RepoThreeCommit')
summary = RepositorySummary('RepoThreeCommit', 'RepoThreeTag',
'3.2.0', '2.2.1', [])
source_info = SourceInfo('BuildThree', summary)
builder.add_repository(repository, source_info)
self.assertEquals(prefix[which], builder.determine_most_common_prefix())
def make_repository_spec(self, name, **kwargs):
"""Create GitRepositorySpec based on the name and configuration.
Args:
git_dir: if supplied then use it, otherwise default under the root path.
origin: if supplied then use it, even if None. Otherwise default
upstream: if supplied then use it, even if None. Otherwise default.
"""
git_dir = kwargs.pop('git_dir', os.path.join(self.__root_source_dir, name))
origin = kwargs.pop('origin', self.AUTO)
upstream = kwargs.pop('upstream', self.AUTO)
check_kwargs_empty(kwargs)
if origin == self.AUTO:
origin = self.determine_origin(name)
if os.path.exists(git_dir):
logging.info('Confirming existing %s matches expectations', git_dir)
existing = self.__git.determine_git_repository_spec(git_dir)
if existing.origin != origin:
raise_and_log_error(
UnexpectedError(
'Repository "{dir}" origin="{have}" expected="{want}"'.format(
dir=git_dir, have=existing.origin, want=origin)))
if upstream == self.AUTO:
upstream = self.determine_upstream_url(name)
return GitRepositorySpec(
name, origin=origin, upstream=upstream, git_dir=git_dir)
def make_repository_spec(self, name, **kwargs):
"""Create GitRepositorySpec based on the name and configuration.
Args:
git_dir: if supplied then use it, otherwise default under the root path.
origin: if supplied then use it, even if None. Otherwise default
upstream: if supplied then use it, even if None. Otherwise default.
kwargs: Additional repository attributes
"""
git_dir = kwargs.pop("git_dir", os.path.join(self.__root_source_dir, name))
origin = kwargs.pop("origin", self.AUTO)
upstream = kwargs.pop("upstream", self.AUTO)
if origin == self.AUTO:
origin = self.determine_origin(name)
if os.path.exists(git_dir):
logging.info("Confirming existing %s matches expectations", git_dir)
existing = self.__git.determine_git_repository_spec(git_dir)
if existing.origin not in [origin, self.__git.determine_pull_url(origin)]:
raise_and_log_error(
UnexpectedError(
'Repository "{dir}" origin="{have}" expected="{want}"'.format(
dir=git_dir, have=existing.origin, want=origin
)
)
)
if upstream == self.AUTO:
upstream = self.determine_upstream_url(name)
return GitRepositorySpec(
name, origin=origin, upstream=upstream, git_dir=git_dir, **kwargs
)
def test_clone_origin(self):
git = self.git
# Make the origin we're going to test the clone against
# This is intentionally different from upstream so that
# we can confirm that upstream is also setup properly.
origin_user = '******'
origin_basedir = os.path.join(self.base_temp_dir, origin_user)
os.makedirs(origin_basedir)
check_subprocess('git -C "{origin_dir}" clone "{upstream}"'.format(
origin_dir=origin_basedir, upstream=self.git_dir))
test_parent = os.path.join(self.base_temp_dir, 'test_clone_origin')
os.makedirs(test_parent)
test_dir = os.path.join(test_parent, TEST_REPO_NAME)
origin_dir = os.path.join(origin_basedir, TEST_REPO_NAME)
repository = GitRepositorySpec(TEST_REPO_NAME,
git_dir=test_dir,
origin=origin_dir,
upstream=self.git_dir)
self.git.clone_repository_to_path(repository)
want_tags = git.query_tag_commits(self.git_dir, TAG_VERSION_PATTERN)
have_tags = git.query_tag_commits(test_dir, TAG_VERSION_PATTERN)
self.assertEqual(want_tags, have_tags)
got = check_subprocess('git -C "{dir}" remote -v'.format(dir=test_dir))
# Upstream repo is configured for pulls, but not for pushes.
self.assertEqual(
'\n'.join([
'origin\t{origin} (fetch)'.format(origin=origin_dir),
'origin\t{origin} (push)'.format(origin=origin_dir),
'upstream\t{upstream} (fetch)'.format(upstream=self.git_dir),
'upstream\tdisabled (push)'
]), got)
reference = git.determine_git_repository_spec(test_dir)
expect = GitRepositorySpec(os.path.basename(origin_dir),
upstream=self.git_dir,
origin=origin_dir,
git_dir=test_dir)
self.assertEqual(expect, reference)
def _do_determine_source_repositories(self):
"""Implements RepositoryCommandProcessor interface."""
# These arent actually used, just the names.
repositories = [
self.source_code_manager.make_repository_spec(name)
for name in SPINNAKER_RUNNABLE_REPOSITORY_NAMES
]
repositories.extend(
[GitRepositorySpec(name) for name in EXTRA_REPO_NAMES])
return repositories
def test_clone_branch(self):
test_parent = os.path.join(self.base_temp_dir, 'test_clone_branch')
os.makedirs(test_parent)
test_dir = os.path.join(test_parent, TEST_REPO_NAME)
repository = GitRepositorySpec(
TEST_REPO_NAME, git_dir=test_dir, origin=self.git_dir)
self.git.clone_repository_to_path(repository, branch=BRANCH_A)
self.assertEqual(BRANCH_A,
self.git.query_local_repository_branch(test_dir))
def test_clone_failure(self):
test_dir = os.path.join(
self.base_temp_dir, 'clone_failure', TEST_REPO_NAME)
os.makedirs(test_dir)
with open(os.path.join(test_dir, 'something'), 'w') as f:
f.write('not empty')
repository = GitRepositorySpec(
TEST_REPO_NAME, git_dir=test_dir, origin=self.git_dir)
regexp = '.* clone .*'
with self.assertRaisesRegexp(Exception, regexp):
self.git.clone_repository_to_path(repository, branch='master')
def test_determine_most_common_prefix(self):
options = self.options
builder = BomBuilder(options, self.scm, MetricsManager.singleton())
self.assertIsNone(builder.determine_most_common_prefix())
prefix = ["http://github.com/one", "/local/source/path/two"]
# Test two vs one in from different repo prefix
# run the test twice changing the ordering the desired prefix is visible.
for which in [0, 1]:
repository = GitRepositorySpec("RepoOne",
origin=prefix[0] + "/RepoOne",
commit_id="RepoOneCommit")
summary = RepositorySummary("RepoOneCommit", "RepoOneTag", "1.2.3",
[])
source_info = SourceInfo("BuildOne", summary)
builder.add_repository(repository, source_info)
self.assertEqual(prefix[0], builder.determine_most_common_prefix())
repository = GitRepositorySpec("RepoTwo",
origin=prefix[which] + "/RepoTwo",
commit_id="RepoTwoCommit")
summary = RepositorySummary("RepoTwoCommit", "RepoTwoTag", "2.2.3",
[])
source_info = SourceInfo("BuildTwo", summary)
builder.add_repository(repository, source_info)
repository = GitRepositorySpec(
"RepoThree",
origin=prefix[1] + "/RepoThree",
commit_id="RepoThreeCommit",
)
summary = RepositorySummary("RepoThreeCommit", "RepoThreeTag",
"3.2.0", [])
source_info = SourceInfo("BuildThree", summary)
builder.add_repository(repository, source_info)
self.assertEqual(prefix[which],
builder.determine_most_common_prefix())
def test_branch_not_found_exception(self):
test_parent = os.path.join(self.base_temp_dir, 'test_bad_branch')
os.makedirs(test_parent)
test_dir = os.path.join(test_parent, TEST_REPO_NAME)
self.assertFalse(os.path.exists(test_dir))
repository = GitRepositorySpec(
TEST_REPO_NAME, git_dir=test_dir, origin=self.git_dir)
branch = 'Bogus'
regexp = r"Branches \['{branch}'\] do not exist in {url}\.".format(
branch=branch, url=self.git_dir)
with self.assertRaisesRegexp(Exception, regexp):
self.git.clone_repository_to_path(repository, branch=branch)
self.assertFalse(os.path.exists(test_dir))
def test_bom_command(self):
"""Make sure when we run "build_bom" we actually get what we meant."""
defaults = vars(make_default_options(self.options))
defaults.update({
'bom_path': 'MY PATH',
'github_owner': 'TestOwner',
'input_dir': 'TestInputRoot'
})
defaults.update({
'bintray_org': 'TestBintrayOrg',
'bintray_debian_repository': 'TestDebianRepo',
'docker_registry': 'TestDockerRegistry',
'publish_gce_image_project': 'TestGceProject'
})
del defaults['github_filesystem_root']
parser = argparse.ArgumentParser()
registry = bomtool_main.make_registry([buildtool.bom_commands], parser,
defaults)
bomtool_main.add_standard_parser_args(parser, defaults)
options = parser.parse_args(['build_bom'])
prefix = 'http://test-domain.com/test-owner'
make_fake = self.patch_method
# When asked to filter the normal bom repos to determine source_repositories
# we'll return our own fake repository as if we configured the original
# command for it. This will also make it easier to test just the one
# repo rather than all, and that there are no assumptions.
mock_filter = make_fake(BuildBomCommand, 'filter_repositories')
test_repository = GitRepositorySpec('clouddriver',
commit_id='CommitA',
origin=prefix + '/TestRepoA')
mock_filter.return_value = [test_repository]
# When the base command ensures the local repository exists, we'll
# intercept that call and do nothing rather than the git checkouts, etc.
make_fake(BranchSourceCodeManager, 'ensure_local_repository')
# When the base command asks for the repository metadata, we'll return
# this hardcoded info, then look for it later in the generated om.
mock_lookup = make_fake(BranchSourceCodeManager, 'lookup_source_info')
summary = RepositorySummary('CommitA', 'TagA', '9.8.7', '44.55.66', [])
source_info = SourceInfo('MyBuildNumber', summary)
mock_lookup.return_value = source_info
# When asked to write the bom out, do nothing.
# We'll verify the bom later when looking at the mock call sequencing.
mock_write = self.patch_function(
'buildtool.bom_commands.write_to_path')
mock_now = self.patch_function('buildtool.bom_commands.now')
mock_now.return_value = datetime.datetime(2018, 1, 2, 3, 4, 5)
factory = registry['build_bom']
command = factory.make_command(options)
command()
# Verify source repositories were filtered
self.assertEquals([test_repository], command.source_repositories)
# Verify that the filter was called with the original bom repos,
# and these repos were coming from the configured github_owner's repo.
bom_repo_list = [
GitRepositorySpec(name,
git_dir=os.path.join('TestInputRoot',
'build_bom', name),
origin='https://%s/TestOwner/%s' %
(options.github_hostname, name),
upstream='https://github.com/spinnaker/' + name)
for name in sorted([
'clouddriver', 'deck', 'echo', 'fiat', 'front50', 'gate',
'igor', 'orca', 'rosco', 'spinnaker', 'spinnaker-monitoring'
])
]
mock_filter.assert_called_once_with(bom_repo_list)
mock_lookup.assert_called_once_with(test_repository)
bom_text, bom_path = mock_write.call_args_list[0][0]
self.assertEquals(bom_path, 'MY PATH')
bom = yaml.load(bom_text)
golden_text = textwrap.dedent("""\
artifactSources:
debianRepository: https://dl.bintray.com/TestBintrayOrg/TestDebianRepo
dockerRegistry: TestDockerRegistry
gitPrefix: http://test-domain.com/test-owner
googleImageProject: TestGceProject
dependencies:
services:
clouddriver:
commit: CommitA
version: 9.8.7-MyBuildNumber
timestamp: '2018-01-02 03:04:05'
version: OptionBranch-OptionBuildNumber
""")
golden_bom = yaml.load(golden_text)
golden_bom['dependencies'] = load_default_bom_dependencies()
for key, value in golden_bom.items():
self.assertEquals(value, bom[key])
# See the License for the specific language governing permissions and
# limitations under the License.
# pylint: disable=missing-docstring
import textwrap
import unittest
from buildtool import GitRepositorySpec, MetricsManager
from buildtool.gradle_support import GradleMetricsUpdater
from test_util import init_runtime
REPOSITORY = GitRepositorySpec("testRepo")
BINTRAY_ERROR_OUTPUT = textwrap.dedent(
"""\
:echo-web:publishBuildDeb
FAILURE: Build completed with 9 failures.
1: Task failed with an exception.
-----------
* What went wrong:
Execution failed for task ':echo-core:bintrayUpload'.
> GOLDEN TESTING ERROR EXAMPLE Could not upload to 'https://api.bintray.com/content/spinnaker-releases/ewiseblatt-maven/echo/1.542.0/com/netflix/spinnaker/echo/echo-core/1.542.0/echo-core-1.542.0.jar': HTTP/1.1 409 Conflict [message:Unable to upload files: An artifact with the path 'com/netflix/spinnaker/echo/echo-core/1.542.0/echo-core-1.542.0.jar' already exists]
* Try:
Run with --info or --debug option to get more log output.
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# pylint: disable=missing-docstring
import textwrap
import unittest
from buildtool import (GitRepositorySpec, MetricsManager)
from buildtool.gradle_support import GradleMetricsUpdater
from test_util import init_runtime
REPOSITORY = GitRepositorySpec('testRepo')
BINTRAY_ERROR_OUTPUT = textwrap.dedent("""\
:echo-web:publishBuildDeb
FAILURE: Build completed with 9 failures.
1: Task failed with an exception.
-----------
* What went wrong:
Execution failed for task ':echo-core:bintrayUpload'.
> GOLDEN TESTING ERROR EXAMPLE Could not upload to 'https://api.bintray.com/content/spinnaker-releases/ewiseblatt-maven/echo/1.542.0/com/netflix/spinnaker/echo/echo-core/1.542.0/echo-core-1.542.0.jar': HTTP/1.1 409 Conflict [message:Unable to upload files: An artifact with the path 'com/netflix/spinnaker/echo/echo-core/1.542.0/echo-core-1.542.0.jar' already exists]
* Try:
Run with --info or --debug option to get more log output.
""")
def test_bom_command(self):
"""Make sure when we run "build_bom" we actually get what we meant."""
defaults = vars(make_default_options(self.options))
defaults.update({
"bom_path": "MY PATH",
"github_owner": "TestOwner",
"input_dir": "TestInputRoot",
})
del defaults["github_repository_root"]
parser = argparse.ArgumentParser()
registry = bomtool_main.make_registry([buildtool.bom_commands], parser,
defaults)
bomtool_main.add_standard_parser_args(parser, defaults)
options = parser.parse_args(["build_bom"])
prefix = "http://test-domain.com/test-owner"
make_fake = self.patch_method
# When asked to filter the normal bom repos to determine source_repositories
# we'll return our own fake repository as if we configured the original
# command for it. This will also make it easier to test just the one
# repo rather than all, and that there are no assumptions.
mock_filter = make_fake(BuildBomCommand, "filter_repositories")
test_repository = GitRepositorySpec("clouddriver",
commit_id="CommitA",
origin=prefix + "/TestRepoA")
mock_filter.return_value = [test_repository]
# When the base command ensures the local repository exists, we'll
# intercept that call and do nothing rather than the git checkouts, etc.
make_fake(BranchSourceCodeManager, "ensure_local_repository")
# When the base command asks for the repository metadata, we'll return
# this hardcoded info, then look for it later in the generated om.
mock_refresh = make_fake(BranchSourceCodeManager,
"refresh_source_info")
summary = RepositorySummary("CommitA", "TagA", "9.8.7", [])
source_info = SourceInfo("MyBuildNumber", summary)
mock_refresh.return_value = source_info
# When asked to write the bom out, do nothing.
# We'll verify the bom later when looking at the mock call sequencing.
mock_write = self.patch_function(
"buildtool.bom_commands.write_to_path")
mock_now = self.patch_function("buildtool.bom_commands.now")
mock_now.return_value = datetime.datetime(2018, 1, 2, 3, 4, 5)
factory = registry["build_bom"]
command = factory.make_command(options)
command()
# Verify source repositories were filtered
self.assertEqual([test_repository], command.source_repositories)
# Verify that the filter was called with the original bom repos,
# and these repos were coming from the configured github_owner's repo.
bom_repo_list = [
GitRepositorySpec(
name,
git_dir=os.path.join("TestInputRoot", "build_bom", name),
origin="https://%s/TestOwner/%s" %
(options.github_hostname, name),
upstream="https://github.com/spinnaker/" + name,
) for name in sorted([
"clouddriver",
"deck",
"echo",
"fiat",
"front50",
"gate",
"igor",
"kayenta",
"orca",
"rosco",
"spinnaker-monitoring",
])
]
mock_filter.assert_called_once_with(bom_repo_list)
mock_refresh.assert_called_once_with(test_repository,
"OptionBuildNumber")
bom_text, bom_path = mock_write.call_args_list[0][0]
self.assertEqual(bom_path, "MY PATH")
bom = yaml.safe_load(bom_text)
golden_text = (textwrap.dedent("""\
artifactSources:
gitPrefix: http://test-domain.com/test-owner
debianRepository: %s
dockerRegistry: %s
googleImageProject: %s
dependencies:
services:
clouddriver:
commit: CommitA
version: 9.8.7
timestamp: '2018-01-02 03:04:05'
version: OptionBuildNumber
""") % (
SPINNAKER_DEBIAN_REPOSITORY,
SPINNAKER_DOCKER_REGISTRY,
SPINNAKER_GOOGLE_IMAGE_PROJECT,
))
golden_bom = yaml.safe_load(golden_text.format())
golden_bom["dependencies"] = load_default_bom_dependencies()
for key, value in golden_bom.items():
self.assertEqual(value, bom[key])
