def list_of_current_bom_meta(min_semver, all_bom_meta):
good_bom_meta = []
for bom_meta in all_bom_meta:
semver = SemanticVersion.make('ignored-' + bom_meta['bom_version'])
if SemanticVersion.compare(semver, min_semver) >= 0:
good_bom_meta.append(bom_meta)
return good_bom_meta
def test_semver_make_valid(self):
tests = [('simple-1.0.0', SemanticVersion('simple', 1, 0, 0)),
('another-10.11.12', SemanticVersion('another', 10, 11, 12))]
for tag, expect in tests:
semver = SemanticVersion.make(tag)
self.assertEqual(semver, expect)
self.assertEqual(tag, semver.to_tag())
self.assertEqual(tag[tag.rfind('-') + 1:], semver.to_version())
def test_maybe_pull_repository_branch(self):
test_root = os.path.join(self.base_temp_dir, 'pulled_test')
options = make_default_options()
options.git_branch = UNTAGGED_BRANCH
options.build_number = 'maybe_pull_branch_buildnum'
scm = BranchSourceCodeManager(options, test_root)
for repository_name, origin in self.ORIGIN_URLS.items():
repository = scm.make_repository_spec(repository_name,
origin=origin,
upstream=None)
scm.ensure_local_repository(repository)
git_dir = repository.git_dir
spec = scm.git.determine_git_repository_spec(git_dir)
self.assertEqual(repository, spec)
in_branch = scm.git.query_local_repository_branch(git_dir)
self.assertEqual(UNTAGGED_BRANCH, in_branch)
summary = scm.git.collect_repository_summary(git_dir)
semver = SemanticVersion.make(BASE_VERSION)
expect_version = semver.next(
SemanticVersion.MINOR_INDEX).to_version()
self.assertEqual(expect_version, summary.version)
def test_pull_bom(self):
input_dir = self.test_root
scm = BomSourceCodeManager(self.options, input_dir, bom=self.golden_bom)
for repo_name in ALL_STANDARD_TEST_BOM_REPO_NAMES:
repository = scm.make_repository_spec(repo_name)
self.assertFalse(os.path.exists(repository.git_dir))
scm.ensure_local_repository(repository)
self.assertTrue(os.path.exists(repository.git_dir))
git_dir = repository.git_dir
spec = scm.git.determine_git_repository_spec(git_dir)
self.assertEquals(repository.name, spec.name)
self.assertEquals(repository.git_dir, spec.git_dir)
self.assertEquals(repository.origin, spec.origin)
self.assertIsNone(spec.upstream_or_none())
repo_name = repository.name
at_commit = scm.git.query_local_repository_commit_id(git_dir)
self.assertEquals(
self.repo_commit_map[repo_name]['ORIGIN'], repository.origin)
self.assertEquals(
self.repo_commit_map[repo_name][PATCH_BRANCH], at_commit)
summary = scm.git.collect_repository_summary(git_dir)
semver = SemanticVersion.make(BASE_VERSION_TAG)
expect_version = semver.next(
SemanticVersion.PATCH_INDEX).to_version()
self.assertEquals(expect_version, summary.version)
def test_pull_bom(self):
input_dir = self.test_root
scm = BomSourceCodeManager(self.options, input_dir, bom=self.golden_bom)
for repo_name in ALL_STANDARD_TEST_BOM_REPO_NAMES:
repository = scm.make_repository_spec(repo_name)
self.assertFalse(os.path.exists(repository.git_dir))
scm.ensure_local_repository(repository)
self.assertTrue(os.path.exists(repository.git_dir))
git_dir = repository.git_dir
spec = scm.git.determine_git_repository_spec(git_dir)
self.assertEqual(repository.name, spec.name)
self.assertEqual(repository.git_dir, spec.git_dir)
self.assertEqual(repository.origin, spec.origin)
self.assertIsNone(spec.upstream_or_none())
repo_name = repository.name
at_commit = scm.git.query_local_repository_commit_id(git_dir)
self.assertEqual(
self.repo_commit_map[repo_name]['ORIGIN'], repository.origin)
self.assertEqual(
self.repo_commit_map[repo_name][PATCH_BRANCH], at_commit)
summary = scm.git.collect_repository_summary(git_dir)
semver = SemanticVersion.make(BASE_VERSION_TAG)
expect_version = semver.next(
SemanticVersion.PATCH_INDEX).to_version()
self.assertEqual(expect_version, summary.version)
def test_semver_make_valid(self):
tests = [('simple-1.0.0', SemanticVersion('simple', 1, 0, 0)),
('another-10.11.12', SemanticVersion('another', 10, 11, 12))]
for tag, expect in tests:
semver = SemanticVersion.make(tag)
self.assertEqual(semver, expect)
self.assertEqual(tag, semver.to_tag())
self.assertEqual(tag[tag.rfind('-') + 1:], semver.to_version())
def bump_spin_patch(git, git_dir, gate_version):
'''Calculates the next spin version from the gate version and previous spin tags.
Spin is coupled to the Gate major and minor version.
Gate is a routing server, so features and breaking changes in Gate
must be reflected in spin since it is a client.
:param git: git support helper class.
:param git_dir: spin git directory.
:param gate_version: gate version to align spin version with in <maj>.<min>.<patch>-<buildnum> format.
:return: (SemanticVersion) Next semver spin version.
'''
gate_version_parts = gate_version.split('-')
if len(gate_version_parts) != 2:
raise_and_log_error(
ValueError('Malformed gate version {}'.format(gate_version)))
# SemanticVersion.make() expects a tag, so formulate the input gate version as a tag.
gate_semver = SemanticVersion.make('version-{}'.format(
gate_version_parts[0]))
tag_pattern = r'version-{maj}.{min}.(\d+)'.format(maj=gate_semver.major,
min=gate_semver.minor)
tag_matcher = re.compile(tag_pattern)
tags = git.fetch_tags(git_dir)
logging.info('searching git tags {} for patterns matching {}'.format(
tags, tag_pattern))
matching_semvers = [
SemanticVersion.make(t) for t in tags if tag_matcher.match(t)
]
logging.info('found matching semvers: {}'.format(matching_semvers))
if matching_semvers:
max_semver = max(matching_semvers)
next_semver = max_semver.next(SemanticVersion.PATCH_INDEX)
patch = next_semver.patch
else:
patch = '0'
# SemanticVersion.make() expects a tag, so formulate the input gate version as a tag.
spin_semver = SemanticVersion.make(
'version-{major}.{minor}.{patch}'.format(major=gate_semver.major,
minor=gate_semver.minor,
patch=patch))
logging.info('calculated next spin patch version: {}'.format(spin_semver))
return spin_semver
def bump_spin_patch(git, git_dir, gate_version):
'''Calculates the next spin version from the gate version and previous spin tags.
Spin is coupled to the Gate major and minor version.
Gate is a routing server, so features and breaking changes in Gate
must be reflected in spin since it is a client.
:param git: git support helper class.
:param git_dir: spin git directory.
:param gate_version: gate version to align spin version with in <maj>.<min>.<patch>-<buildnum> format.
:return: (SemanticVersion) Next semver spin version.
'''
gate_version_parts = gate_version.split('-')
if len(gate_version_parts) != 2:
raise_and_log_error(
ValueError('Malformed gate version {}'.format(gate_version)))
# SemanticVersion.make() expects a tag, so formulate the input gate version as a tag.
gate_semver = SemanticVersion.make('version-{}'.format(gate_version_parts[0]))
tag_pattern = r'version-{maj}.{min}.(\d+)'.format(maj=gate_semver.major,
min=gate_semver.minor)
tag_matcher = re.compile(tag_pattern)
tags = git.fetch_tags(git_dir)
logging.info('searching git tags {} for patterns matching {}'.format(tags, tag_pattern))
matching_semvers = [SemanticVersion.make(t) for t in tags if tag_matcher.match(t)]
logging.info('found matching semvers: {}'.format(matching_semvers))
if matching_semvers:
max_semver = max(matching_semvers)
next_semver = max_semver.next(SemanticVersion.PATCH_INDEX)
patch = next_semver.patch
else:
patch = '0'
# SemanticVersion.make() expects a tag, so formulate the input gate version as a tag.
spin_semver = SemanticVersion.make('version-{major}.{minor}.{patch}'
.format(major=gate_semver.major,
minor=gate_semver.minor,
patch=patch))
logging.info('calculated next spin patch version: {}'.format(spin_semver))
return spin_semver
def test_semver_next(self):
semver = SemanticVersion('A', 1, 2, 3)
tests = [
(SemanticVersion.TAG_INDEX, SemanticVersion('B', 1, 2, 3), None),
(None, SemanticVersion('A', 1, 2, 3), None),
(SemanticVersion.MAJOR_INDEX,
SemanticVersion('A', 2, 2, 3),
SemanticVersion('A', 2, 0, 0)), # next major index to semver
(SemanticVersion.MINOR_INDEX,
SemanticVersion('A', 1, 3, 3),
SemanticVersion('A', 1, 3, 0)), # next minor index to semver
(SemanticVersion.PATCH_INDEX,
SemanticVersion('A', 1, 2, 4),
SemanticVersion('A', 1, 2, 4)), # next patch index to semver
]
for expect_index, test, next_semver in tests:
self.assertEqual(expect_index, semver.most_significant_diff_index(test))
self.assertEqual(expect_index, test.most_significant_diff_index(semver))
if expect_index is not None and expect_index > SemanticVersion.TAG_INDEX:
self.assertEqual(next_semver, semver.next(expect_index))
def most_recent_version(self, name, versions):
"""Find the most recent version built."""
if not versions:
return None
raw_versions = set([version.split('-')[0] for version in versions])
sem_vers = []
for text in raw_versions:
try:
sem_vers.append(SemanticVersion.make('version-' + text))
except Exception as ex:
bad_list = self.__invalid_versions.get(name, [])
bad_list.append(text)
self.__invalid_versions[name] = bad_list
logging.error('Ignoring invalid %s version "%s": %s', name, text, ex)
return sorted(sem_vers, cmp=SemanticVersion.compare)[-1].to_version()
def audit_artifacts(self):
self.audit_bom_services(self.__all_released_boms, 'released')
self.audit_bom_services(self.__unreleased_boms, 'unreleased')
self.audit_package(
'jar', self.__jar_versions, self.__unused_jars)
self.audit_package(
'debian', self.__debian_versions, self.__unused_debians)
self.audit_package(
'container', self.__container_versions, self.__unused_containers)
self.audit_package(
'image',
self.__gce_image_versions, self.__unused_gce_images)
def maybe_write_log(what, data):
if not data:
return
path = os.path.join(self.get_output_dir(), 'audit_' + what + '.yml')
logging.info('Writing %s', path)
write_to_path(
yaml.dump(data, allow_unicode=True, default_flow_style=False),
path)
confirmed_boms = self.__all_bom_versions - set(self.__invalid_boms.keys())
unchecked_releases = [
key
for key in self.__all_bom_versions
if (CollectBomVersions.RELEASED_VERSION_MATCHER.match(key)
and SemanticVersion.compare(SemanticVersion.make('ignored-' + key),
self.__min_semver) < 0)]
invalid_releases = {
key: bom
for key, bom in self.__invalid_boms.items()
if (CollectBomVersions.RELEASED_VERSION_MATCHER.match(key)
and SemanticVersion.compare(SemanticVersion.make('ignored-' + key),
self.__min_semver) >= 0)}
confirmed_releases = [
key
for key in confirmed_boms
if (CollectBomVersions.RELEASED_VERSION_MATCHER.match(key)
and SemanticVersion.compare(SemanticVersion.make('ignored-' + key),
self.__min_semver) >= 0)]
maybe_write_log('missing_debians', self.__missing_debians)
maybe_write_log('missing_jars', self.__missing_jars)
maybe_write_log('missing_containers', self.__missing_containers)
maybe_write_log('missing_images', self.__missing_images)
maybe_write_log('found_debians', self.__found_debians)
maybe_write_log('found_jars', self.__found_jars)
maybe_write_log('found_containers', self.__found_containers)
maybe_write_log('found_images', self.__found_images)
maybe_write_log('unused_debians', self.__unused_debians)
maybe_write_log('unused_jars', self.__unused_jars)
maybe_write_log('unused_containers', self.__unused_containers)
maybe_write_log('unused_images', self.__unused_gce_images)
maybe_write_log('invalid_boms', self.__invalid_boms)
maybe_write_log('confirmed_boms', sorted(list(confirmed_boms)))
maybe_write_log('confirmed_releases', sorted(list(confirmed_releases)))
maybe_write_log('invalid_versions', self.__invalid_versions)
maybe_write_log('invalid_releases', invalid_releases)
maybe_write_log('unchecked_releases', unchecked_releases)
def __init__(self, factory, options, **kwargs):
if options.prune_min_buildnum_prefix is not None:
# Typically numeric so is interpreted as number from yaml
options.prune_min_buildnum_prefix = str(options.prune_min_buildnum_prefix)
super(AuditArtifactVersions, self).__init__(factory, options, **kwargs)
base_path = os.path.dirname(self.get_output_dir())
self.__init_bintray_versions_helper(base_path)
min_version = options.min_audit_bom_version or '0.0.0'
min_parts = min_version.split('.')
if len(min_parts) < 3:
min_version += '.0' * (3 - len(min_parts))
self.__min_semver = SemanticVersion.make('ignored-' + min_version)
bom_data_dir = os.path.join(base_path, 'collect_bom_versions')
path = os.path.join(bom_data_dir, 'released_bom_service_map.yml')
check_path_exists(path, 'released bom analysis')
with open(path, 'r') as stream:
self.__all_released_boms = {} # forever
self.__current_released_boms = {} # since min_version to audit
for service, versions in yaml.load(stream.read()).items():
if not versions:
# e.g. this service has not yet been released.
logging.info('No versions for service=%s', service)
continue
self.__all_released_boms[service] = versions
self.__current_released_boms[service] = versions
stripped_versions = self.__remove_old_bom_versions(
self.__min_semver, versions)
if stripped_versions:
self.__current_released_boms[service] = stripped_versions
path = os.path.join(bom_data_dir, 'unreleased_bom_service_map.yml')
check_path_exists(path, 'unreleased bom analysis')
with open(path, 'r') as stream:
self.__unreleased_boms = yaml.load(stream.read())
self.__only_bad_and_invalid_boms = False
self.__all_bom_versions = self.__extract_all_bom_versions(
self.__all_released_boms)
self.__all_bom_versions.update(
self.__extract_all_bom_versions(self.__unreleased_boms))
self.__missing_debians = {}
self.__missing_jars = {}
self.__missing_containers = {}
self.__missing_images = {}
self.__found_debians = {}
self.__found_jars = {}
self.__found_containers = {}
self.__found_images = {}
self.__unused_jars = {}
self.__unused_debians = {}
self.__unused_containers = {}
self.__unused_gce_images = {}
self.__invalid_boms = {}
self.__confirmed_boms = set([])
self.__prune_boms = []
self.__prune_jars = {}
self.__prune_debians = {}
self.__prune_containers = {}
self.__prune_gce_images = {}
self.__invalid_versions = {}
def test_semver_sort(self):
versions = [
SemanticVersion.make('version-1.9.7'),
SemanticVersion.make('version-9.8.7'),
SemanticVersion.make('version-11.0.0'),
SemanticVersion.make('version-3.10.2'),
SemanticVersion.make('version-3.0.4'),
SemanticVersion.make('version-3.2.2'),
SemanticVersion.make('version-3.2.0'),
SemanticVersion.make('version-3.2.1'),
]
got = sorted(versions)
expect = [
SemanticVersion.make('version-1.9.7'),
SemanticVersion.make('version-3.0.4'),
SemanticVersion.make('version-3.2.0'),
SemanticVersion.make('version-3.2.1'),
SemanticVersion.make('version-3.2.2'),
SemanticVersion.make('version-3.10.2'),
SemanticVersion.make('version-9.8.7'),
SemanticVersion.make('version-11.0.0'),
]
self.assertEquals(expect, got)
def test_semver_next(self):
semver = SemanticVersion('A', 1, 2, 3)
tests = [
(SemanticVersion.TAG_INDEX, SemanticVersion('B', 1, 2, 3), None),
(None, SemanticVersion('A', 1, 2, 3), None),
(SemanticVersion.MAJOR_INDEX, SemanticVersion('A', 2, 2, 3),
SemanticVersion('A', 2, 0, 0)), # next major index to semver
(SemanticVersion.MINOR_INDEX, SemanticVersion('A', 1, 3, 3),
SemanticVersion('A', 1, 3, 0)), # next minor index to semver
(SemanticVersion.PATCH_INDEX, SemanticVersion('A', 1, 2, 4),
SemanticVersion('A', 1, 2, 4)), # next patch index to semver
]
for expect_index, test, next_semver in tests:
self.assertEqual(expect_index,
semver.most_significant_diff_index(test))
self.assertEqual(expect_index,
test.most_significant_diff_index(semver))
if expect_index is not None and expect_index > SemanticVersion.TAG_INDEX:
self.assertEqual(next_semver, semver.next(expect_index))
def test_semver_sort(self):
versions = [
SemanticVersion.make('version-1.9.7'),
SemanticVersion.make('version-9.8.7'),
SemanticVersion.make('version-11.0.0'),
SemanticVersion.make('version-3.10.2'),
SemanticVersion.make('version-3.0.4'),
SemanticVersion.make('version-3.2.2'),
SemanticVersion.make('version-3.2.0'),
SemanticVersion.make('version-3.2.1'),
]
got = sorted(versions)
expect = [
SemanticVersion.make('version-1.9.7'),
SemanticVersion.make('version-3.0.4'),
SemanticVersion.make('version-3.2.0'),
SemanticVersion.make('version-3.2.1'),
SemanticVersion.make('version-3.2.2'),
SemanticVersion.make('version-3.10.2'),
SemanticVersion.make('version-9.8.7'),
SemanticVersion.make('version-11.0.0'),
]
self.assertEquals(expect, got)