Exemple #1
0
 def list_of_current_bom_meta(min_semver, all_bom_meta):
   good_bom_meta = []
   for bom_meta in all_bom_meta:
     semver = SemanticVersion.make('ignored-' + bom_meta['bom_version'])
     if SemanticVersion.compare(semver, min_semver) >= 0:
       good_bom_meta.append(bom_meta)
   return good_bom_meta
Exemple #2
0
 def test_semver_make_valid(self):
     tests = [('simple-1.0.0', SemanticVersion('simple', 1, 0, 0)),
              ('another-10.11.12', SemanticVersion('another', 10, 11, 12))]
     for tag, expect in tests:
         semver = SemanticVersion.make(tag)
         self.assertEqual(semver, expect)
         self.assertEqual(tag, semver.to_tag())
         self.assertEqual(tag[tag.rfind('-') + 1:], semver.to_version())
Exemple #3
0
    def test_maybe_pull_repository_branch(self):
        test_root = os.path.join(self.base_temp_dir, 'pulled_test')
        options = make_default_options()
        options.git_branch = UNTAGGED_BRANCH
        options.build_number = 'maybe_pull_branch_buildnum'
        scm = BranchSourceCodeManager(options, test_root)

        for repository_name, origin in self.ORIGIN_URLS.items():
            repository = scm.make_repository_spec(repository_name,
                                                  origin=origin,
                                                  upstream=None)
            scm.ensure_local_repository(repository)

            git_dir = repository.git_dir
            spec = scm.git.determine_git_repository_spec(git_dir)
            self.assertEqual(repository, spec)

            in_branch = scm.git.query_local_repository_branch(git_dir)
            self.assertEqual(UNTAGGED_BRANCH, in_branch)

            summary = scm.git.collect_repository_summary(git_dir)
            semver = SemanticVersion.make(BASE_VERSION)
            expect_version = semver.next(
                SemanticVersion.MINOR_INDEX).to_version()

            self.assertEqual(expect_version, summary.version)
Exemple #4
0
  def test_pull_bom(self):
    input_dir = self.test_root
    scm = BomSourceCodeManager(self.options, input_dir, bom=self.golden_bom)
    for repo_name in ALL_STANDARD_TEST_BOM_REPO_NAMES:
      repository = scm.make_repository_spec(repo_name)
      self.assertFalse(os.path.exists(repository.git_dir))
      scm.ensure_local_repository(repository)
      self.assertTrue(os.path.exists(repository.git_dir))

      git_dir = repository.git_dir
      spec = scm.git.determine_git_repository_spec(git_dir)
      self.assertEquals(repository.name, spec.name)
      self.assertEquals(repository.git_dir, spec.git_dir)
      self.assertEquals(repository.origin, spec.origin)
      self.assertIsNone(spec.upstream_or_none())

      repo_name = repository.name
      at_commit = scm.git.query_local_repository_commit_id(git_dir)
      self.assertEquals(
          self.repo_commit_map[repo_name]['ORIGIN'], repository.origin)
      self.assertEquals(
          self.repo_commit_map[repo_name][PATCH_BRANCH], at_commit)

      summary = scm.git.collect_repository_summary(git_dir)
      semver = SemanticVersion.make(BASE_VERSION_TAG)
      expect_version = semver.next(
          SemanticVersion.PATCH_INDEX).to_version()
      self.assertEquals(expect_version, summary.version)
Exemple #5
0
  def test_pull_bom(self):
    input_dir = self.test_root
    scm = BomSourceCodeManager(self.options, input_dir, bom=self.golden_bom)
    for repo_name in ALL_STANDARD_TEST_BOM_REPO_NAMES:
      repository = scm.make_repository_spec(repo_name)
      self.assertFalse(os.path.exists(repository.git_dir))
      scm.ensure_local_repository(repository)
      self.assertTrue(os.path.exists(repository.git_dir))

      git_dir = repository.git_dir
      spec = scm.git.determine_git_repository_spec(git_dir)
      self.assertEqual(repository.name, spec.name)
      self.assertEqual(repository.git_dir, spec.git_dir)
      self.assertEqual(repository.origin, spec.origin)
      self.assertIsNone(spec.upstream_or_none())

      repo_name = repository.name
      at_commit = scm.git.query_local_repository_commit_id(git_dir)
      self.assertEqual(
          self.repo_commit_map[repo_name]['ORIGIN'], repository.origin)
      self.assertEqual(
          self.repo_commit_map[repo_name][PATCH_BRANCH], at_commit)

      summary = scm.git.collect_repository_summary(git_dir)
      semver = SemanticVersion.make(BASE_VERSION_TAG)
      expect_version = semver.next(
          SemanticVersion.PATCH_INDEX).to_version()
      self.assertEqual(expect_version, summary.version)
 def test_semver_make_valid(self):
   tests = [('simple-1.0.0', SemanticVersion('simple', 1, 0, 0)),
            ('another-10.11.12', SemanticVersion('another', 10, 11, 12))]
   for tag, expect in tests:
     semver = SemanticVersion.make(tag)
     self.assertEqual(semver, expect)
     self.assertEqual(tag, semver.to_tag())
     self.assertEqual(tag[tag.rfind('-') + 1:], semver.to_version())
def bump_spin_patch(git, git_dir, gate_version):
    '''Calculates the next spin version from the gate version and previous spin tags.

  Spin is coupled to the Gate major and minor version.
  Gate is a routing server, so features and breaking changes in Gate
  must be reflected in spin since it is a client.

  :param git: git support helper class.
  :param git_dir: spin git directory.
  :param gate_version: gate version to align spin version with in <maj>.<min>.<patch>-<buildnum> format.
  :return: (SemanticVersion) Next semver spin version.
  '''
    gate_version_parts = gate_version.split('-')
    if len(gate_version_parts) != 2:
        raise_and_log_error(
            ValueError('Malformed gate version {}'.format(gate_version)))

    # SemanticVersion.make() expects a tag, so formulate the input gate version as a tag.
    gate_semver = SemanticVersion.make('version-{}'.format(
        gate_version_parts[0]))
    tag_pattern = r'version-{maj}.{min}.(\d+)'.format(maj=gate_semver.major,
                                                      min=gate_semver.minor)
    tag_matcher = re.compile(tag_pattern)
    tags = git.fetch_tags(git_dir)

    logging.info('searching git tags {} for patterns matching {}'.format(
        tags, tag_pattern))
    matching_semvers = [
        SemanticVersion.make(t) for t in tags if tag_matcher.match(t)
    ]
    logging.info('found matching semvers: {}'.format(matching_semvers))
    if matching_semvers:
        max_semver = max(matching_semvers)
        next_semver = max_semver.next(SemanticVersion.PATCH_INDEX)
        patch = next_semver.patch
    else:
        patch = '0'

    # SemanticVersion.make() expects a tag, so formulate the input gate version as a tag.
    spin_semver = SemanticVersion.make(
        'version-{major}.{minor}.{patch}'.format(major=gate_semver.major,
                                                 minor=gate_semver.minor,
                                                 patch=patch))
    logging.info('calculated next spin patch version: {}'.format(spin_semver))
    return spin_semver
Exemple #8
0
def bump_spin_patch(git, git_dir, gate_version):
  '''Calculates the next spin version from the gate version and previous spin tags.

  Spin is coupled to the Gate major and minor version.
  Gate is a routing server, so features and breaking changes in Gate
  must be reflected in spin since it is a client.

  :param git: git support helper class.
  :param git_dir: spin git directory.
  :param gate_version: gate version to align spin version with in <maj>.<min>.<patch>-<buildnum> format.
  :return: (SemanticVersion) Next semver spin version.
  '''
  gate_version_parts = gate_version.split('-')
  if len(gate_version_parts) != 2:
    raise_and_log_error(
        ValueError('Malformed gate version {}'.format(gate_version)))

  # SemanticVersion.make() expects a tag, so formulate the input gate version as a tag.
  gate_semver = SemanticVersion.make('version-{}'.format(gate_version_parts[0]))
  tag_pattern = r'version-{maj}.{min}.(\d+)'.format(maj=gate_semver.major,
                                                    min=gate_semver.minor)
  tag_matcher = re.compile(tag_pattern)
  tags = git.fetch_tags(git_dir)

  logging.info('searching git tags {} for patterns matching {}'.format(tags, tag_pattern))
  matching_semvers = [SemanticVersion.make(t) for t in tags if tag_matcher.match(t)]
  logging.info('found matching semvers: {}'.format(matching_semvers))
  if matching_semvers:
    max_semver = max(matching_semvers)
    next_semver = max_semver.next(SemanticVersion.PATCH_INDEX)
    patch = next_semver.patch
  else:
    patch = '0'

  # SemanticVersion.make() expects a tag, so formulate the input gate version as a tag.
  spin_semver = SemanticVersion.make('version-{major}.{minor}.{patch}'
                                     .format(major=gate_semver.major,
                                             minor=gate_semver.minor,
                                             patch=patch))
  logging.info('calculated next spin patch version: {}'.format(spin_semver))
  return spin_semver
  def test_semver_next(self):
    semver = SemanticVersion('A', 1, 2, 3)
    tests = [
        (SemanticVersion.TAG_INDEX, SemanticVersion('B', 1, 2, 3), None),
        (None, SemanticVersion('A', 1, 2, 3), None),

        (SemanticVersion.MAJOR_INDEX,
         SemanticVersion('A', 2, 2, 3),
         SemanticVersion('A', 2, 0, 0)),  # next major index to semver

        (SemanticVersion.MINOR_INDEX,
         SemanticVersion('A', 1, 3, 3),
         SemanticVersion('A', 1, 3, 0)),  # next minor index to semver

        (SemanticVersion.PATCH_INDEX,
         SemanticVersion('A', 1, 2, 4),
         SemanticVersion('A', 1, 2, 4)),  # next patch index to semver
    ]
    for expect_index, test, next_semver in tests:
      self.assertEqual(expect_index, semver.most_significant_diff_index(test))
      self.assertEqual(expect_index, test.most_significant_diff_index(semver))
      if expect_index is not None and expect_index > SemanticVersion.TAG_INDEX:
        self.assertEqual(next_semver, semver.next(expect_index))
Exemple #10
0
 def most_recent_version(self, name, versions):
   """Find the most recent version built."""
   if not versions:
     return None
   raw_versions = set([version.split('-')[0] for version in versions])
   sem_vers = []
   for text in raw_versions:
     try:
       sem_vers.append(SemanticVersion.make('version-' + text))
     except Exception as ex:
       bad_list = self.__invalid_versions.get(name, [])
       bad_list.append(text)
       self.__invalid_versions[name] = bad_list
       logging.error('Ignoring invalid %s version "%s": %s', name, text, ex)
   return sorted(sem_vers, cmp=SemanticVersion.compare)[-1].to_version()
Exemple #11
0
  def audit_artifacts(self):
    self.audit_bom_services(self.__all_released_boms, 'released')
    self.audit_bom_services(self.__unreleased_boms, 'unreleased')
    self.audit_package(
        'jar', self.__jar_versions, self.__unused_jars)
    self.audit_package(
        'debian', self.__debian_versions, self.__unused_debians)
    self.audit_package(
        'container', self.__container_versions, self.__unused_containers)
    self.audit_package(
        'image',
        self.__gce_image_versions, self.__unused_gce_images)

    def maybe_write_log(what, data):
      if not data:
        return
      path = os.path.join(self.get_output_dir(), 'audit_' + what + '.yml')
      logging.info('Writing %s', path)
      write_to_path(
          yaml.dump(data, allow_unicode=True, default_flow_style=False),
          path)

    confirmed_boms = self.__all_bom_versions - set(self.__invalid_boms.keys())
    unchecked_releases = [
        key
        for key in self.__all_bom_versions
        if (CollectBomVersions.RELEASED_VERSION_MATCHER.match(key)
            and SemanticVersion.compare(SemanticVersion.make('ignored-' + key),
                                        self.__min_semver) < 0)]

    invalid_releases = {
        key: bom
        for key, bom in self.__invalid_boms.items()
        if (CollectBomVersions.RELEASED_VERSION_MATCHER.match(key)
            and SemanticVersion.compare(SemanticVersion.make('ignored-' + key),
                                        self.__min_semver) >= 0)}
    confirmed_releases = [
        key
        for key in confirmed_boms
        if (CollectBomVersions.RELEASED_VERSION_MATCHER.match(key)
            and SemanticVersion.compare(SemanticVersion.make('ignored-' + key),
                                        self.__min_semver) >= 0)]

    maybe_write_log('missing_debians', self.__missing_debians)
    maybe_write_log('missing_jars', self.__missing_jars)
    maybe_write_log('missing_containers', self.__missing_containers)
    maybe_write_log('missing_images', self.__missing_images)
    maybe_write_log('found_debians', self.__found_debians)
    maybe_write_log('found_jars', self.__found_jars)
    maybe_write_log('found_containers', self.__found_containers)
    maybe_write_log('found_images', self.__found_images)
    maybe_write_log('unused_debians', self.__unused_debians)
    maybe_write_log('unused_jars', self.__unused_jars)
    maybe_write_log('unused_containers', self.__unused_containers)
    maybe_write_log('unused_images', self.__unused_gce_images)
    maybe_write_log('invalid_boms', self.__invalid_boms)
    maybe_write_log('confirmed_boms', sorted(list(confirmed_boms)))
    maybe_write_log('confirmed_releases', sorted(list(confirmed_releases)))
    maybe_write_log('invalid_versions', self.__invalid_versions)
    maybe_write_log('invalid_releases', invalid_releases)
    maybe_write_log('unchecked_releases', unchecked_releases)
Exemple #12
0
  def __init__(self, factory, options, **kwargs):
    if options.prune_min_buildnum_prefix is not None:
      # Typically numeric so is interpreted as number from yaml
      options.prune_min_buildnum_prefix = str(options.prune_min_buildnum_prefix)

    super(AuditArtifactVersions, self).__init__(factory, options, **kwargs)
    base_path = os.path.dirname(self.get_output_dir())
    self.__init_bintray_versions_helper(base_path)

    min_version = options.min_audit_bom_version or '0.0.0'
    min_parts = min_version.split('.')
    if len(min_parts) < 3:
      min_version += '.0' * (3 - len(min_parts))
    self.__min_semver = SemanticVersion.make('ignored-' + min_version)

    bom_data_dir = os.path.join(base_path, 'collect_bom_versions')
    path = os.path.join(bom_data_dir, 'released_bom_service_map.yml')
    check_path_exists(path, 'released bom analysis')
    with open(path, 'r') as stream:
      self.__all_released_boms = {}      # forever
      self.__current_released_boms = {}  # since min_version to audit
      for service, versions in yaml.load(stream.read()).items():
        if not versions:
          # e.g. this service has not yet been released.
          logging.info('No versions for service=%s', service)
          continue

        self.__all_released_boms[service] = versions
        self.__current_released_boms[service] = versions
        stripped_versions = self.__remove_old_bom_versions(
            self.__min_semver, versions)
        if stripped_versions:
          self.__current_released_boms[service] = stripped_versions

    path = os.path.join(bom_data_dir, 'unreleased_bom_service_map.yml')
    check_path_exists(path, 'unreleased bom analysis')
    with open(path, 'r') as stream:
      self.__unreleased_boms = yaml.load(stream.read())

    self.__only_bad_and_invalid_boms = False
    self.__all_bom_versions = self.__extract_all_bom_versions(
        self.__all_released_boms)
    self.__all_bom_versions.update(
        self.__extract_all_bom_versions(self.__unreleased_boms))

    self.__missing_debians = {}
    self.__missing_jars = {}
    self.__missing_containers = {}
    self.__missing_images = {}
    self.__found_debians = {}
    self.__found_jars = {}
    self.__found_containers = {}
    self.__found_images = {}
    self.__unused_jars = {}
    self.__unused_debians = {}
    self.__unused_containers = {}
    self.__unused_gce_images = {}
    self.__invalid_boms = {}
    self.__confirmed_boms = set([])
    self.__prune_boms = []
    self.__prune_jars = {}
    self.__prune_debians = {}
    self.__prune_containers = {}
    self.__prune_gce_images = {}
    self.__invalid_versions = {}
  def test_semver_sort(self):
    versions = [
        SemanticVersion.make('version-1.9.7'),
        SemanticVersion.make('version-9.8.7'),
        SemanticVersion.make('version-11.0.0'),
        SemanticVersion.make('version-3.10.2'),
        SemanticVersion.make('version-3.0.4'),
        SemanticVersion.make('version-3.2.2'),
        SemanticVersion.make('version-3.2.0'),
        SemanticVersion.make('version-3.2.1'),
    ]

    got = sorted(versions)
    expect = [
        SemanticVersion.make('version-1.9.7'),
        SemanticVersion.make('version-3.0.4'),
        SemanticVersion.make('version-3.2.0'),
        SemanticVersion.make('version-3.2.1'),
        SemanticVersion.make('version-3.2.2'),
        SemanticVersion.make('version-3.10.2'),
        SemanticVersion.make('version-9.8.7'),
        SemanticVersion.make('version-11.0.0'),
    ]
    self.assertEquals(expect, got)
Exemple #14
0
 def test_semver_next(self):
     semver = SemanticVersion('A', 1, 2, 3)
     tests = [
         (SemanticVersion.TAG_INDEX, SemanticVersion('B', 1, 2, 3), None),
         (None, SemanticVersion('A', 1, 2, 3), None),
         (SemanticVersion.MAJOR_INDEX, SemanticVersion('A', 2, 2, 3),
          SemanticVersion('A', 2, 0, 0)),  # next major index to semver
         (SemanticVersion.MINOR_INDEX, SemanticVersion('A', 1, 3, 3),
          SemanticVersion('A', 1, 3, 0)),  # next minor index to semver
         (SemanticVersion.PATCH_INDEX, SemanticVersion('A', 1, 2, 4),
          SemanticVersion('A', 1, 2, 4)),  # next patch index to semver
     ]
     for expect_index, test, next_semver in tests:
         self.assertEqual(expect_index,
                          semver.most_significant_diff_index(test))
         self.assertEqual(expect_index,
                          test.most_significant_diff_index(semver))
         if expect_index is not None and expect_index > SemanticVersion.TAG_INDEX:
             self.assertEqual(next_semver, semver.next(expect_index))
Exemple #15
0
    def test_semver_sort(self):
        versions = [
            SemanticVersion.make('version-1.9.7'),
            SemanticVersion.make('version-9.8.7'),
            SemanticVersion.make('version-11.0.0'),
            SemanticVersion.make('version-3.10.2'),
            SemanticVersion.make('version-3.0.4'),
            SemanticVersion.make('version-3.2.2'),
            SemanticVersion.make('version-3.2.0'),
            SemanticVersion.make('version-3.2.1'),
        ]

        got = sorted(versions)
        expect = [
            SemanticVersion.make('version-1.9.7'),
            SemanticVersion.make('version-3.0.4'),
            SemanticVersion.make('version-3.2.0'),
            SemanticVersion.make('version-3.2.1'),
            SemanticVersion.make('version-3.2.2'),
            SemanticVersion.make('version-3.10.2'),
            SemanticVersion.make('version-9.8.7'),
            SemanticVersion.make('version-11.0.0'),
        ]
        self.assertEquals(expect, got)