def test_get_projects(self): f1 = """ project-templates: default: uri: https://bitbucket.com/%(name)s branches: - master - ocata gitweb: https://bitbucket.com/%(name)s/commit/%%(sha)s projects: Barbican: logo: https://logo.png description: Credentials storage repos: openstack/barbican: template: default openstack/python-barbicanclient: template: default Swift: repos: openstack/swift: template: default openstack/python-swiftclient: template: default description: The Swift client forks: 10 watchers: 20 stars: 30 branches: - master - 1.0-dev - 2.0-dev """ self.simple_provision(data=f1) p = projects.Projects(db_path=self.db, con=self.con) self.assertEqual(len(p.get_projects()['Swift']['refs']), 5) self.assertEqual(len(p.get_projects()['Barbican']['refs']), 4) branches = [ ref['branch'] for ref in p.get_projects()["Swift"]['refs'] if ref['name'] == 'openstack/python-swiftclient' ] self.assertListEqual(branches, ['master', '1.0-dev', '2.0-dev']) p = projects.Projects(con=self.con) self.assertEqual(len(p.get_projects()['Swift']['refs']), 5) ref = [ r for r in p.get_projects()['Swift']['refs'] if r['name'] == 'openstack/python-swiftclient' ][0] self.assertEqual(ref['description'], 'The Swift client') self.assertEqual(ref['forks'], 10) self.assertEqual(ref['watchers'], 20) self.assertEqual(ref['stars'], 30) # Check we get only the field name ret = p.get_projects(source=['name']) self.assertEqual(len(ret['Swift']), 1) self.assertIn('name', ret['Swift'])
def test_get_projects_from_references(self): f1 = """ project-templates: default: uri: https://bitbucket.com/%(name)s branches: - master gitweb: https://bitbucket.com/%(name)s/commit/%%(sha)s projects: Barbican: repos: openstack/barbican: template: default Swift: repos: openstack/swift: template: default Nova: repos: openstack/nova: template: default """ self.simple_provision(data=f1) p = projects.Projects(db_path=self.db, con=self.con) refs = [('https://bitbucket.com/openstack/barbican:' 'openstack/barbican:master'), ('https://bitbucket.com/openstack/swift:' 'openstack/swift:master')] ret = p.get_projects_from_references(refs) self.assertIn('Swift', ret) self.assertIn('Barbican', ret) self.assertEqual(len(ret), 2)
def test_get_gitweb_link(self): self.simple_provision() p = projects.Projects(db_path=self.db, con=self.con) link = p.get_gitweb_link(('https://bitbucket.com/openstack/barbican:' 'openstack/barbican:master')) self.assertEqual( link, 'https://bitbucket.com/openstack/barbican/commit/%(sha)s') link = p.get_gitweb_link('notexists') self.assertEqual(link, '')
def test_get_references_from_tags(self): f1 = """ project-templates: default: uri: https://bitbucket.com/%(name)s branches: - master - ocata gitweb: https://bitbucket.com/%(name)s/commit/%%(sha)s tags: - openstack - cloud projects: Barbican: repos: openstack/barbican: template: default tags: - credentials - server openstack/python-barbicanclient: template: default tags: - credentials - client Swift: repos: openstack/swift: template: default tags: - storage - server - object-storage openstack/python-swiftclient: template: default tags: - storage - client """ self.simple_provision(data=f1) p = projects.Projects(db_path=self.db, con=self.con) refs = p.get_references_from_tags(['credentials']) self.assertIn( { 'fullrid': ('https://bitbucket.com/openstack/python-barbicanclient:' 'openstack/python-barbicanclient:ocata'), 'name': 'openstack/python-barbicanclient', 'branch': 'ocata' }, refs) self.assertEqual(len(refs), 4) refs = p.get_references_from_tags(['credentials', 'object-storage']) self.assertEqual(len(refs), 6)
def test_get_tags(self): f1 = """ project-templates: default: uri: https://bitbucket.com/%(name)s branches: - master - ocata gitweb: https://bitbucket.com/%(name)s/commit/%%(sha)s tags: - openstack - cloud projects: Barbican: repos: openstack/barbican: template: default tags: - credentials - server openstack/python-barbicanclient: template: default tags: - credentials - client Swift: repos: openstack/swift: template: default tags: - storage - server openstack/python-swiftclient: template: default tags: - storage - client """ files = {'f1.yaml': f1} db = self.create_db(files) index.conf['db_default_file'] = None p = projects.Projects(db_path=db) tags = p.get_tags() self.assertEqual(len(tags['credentials']['repos']), 4) self.assertEqual(len(tags['storage']['repos']), 4) self.assertEqual(len(tags.keys()), 6) for tag in ('openstack', 'cloud', 'client', 'server', 'credentials', 'storage'): self.assertIn(tag, tags.keys())
def test_get_tags(self): f1 = """ project-templates: default: uri: https://bitbucket.com/%(name)s branches: - master - ocata gitweb: https://bitbucket.com/%(name)s/commit/%%(sha)s tags: - openstack - cloud projects: Barbican: repos: openstack/barbican: template: default tags: - credentials - server openstack/python-barbicanclient: template: default tags: - credentials - client Swift: repos: openstack/swift: template: default tags: - storage - server openstack/python-swiftclient: template: default tags: - storage - client """ self.simple_provision(data=f1) p = projects.Projects(db_path=self.db, con=self.con) tags = p.get_tags() for tag in ('openstack', 'cloud', 'client', 'server', 'credentials', 'storage'): self.assertIn(tag, tags)
def simple_provision(self, data=None): f1 = """ project-templates: default: uri: https://bitbucket.com/%(name)s branches: - master gitweb: https://bitbucket.com/%(name)s/commit/%%(sha)s projects: Barbican: repos: openstack/barbican: template: default """ files = {'f1.yaml': data or f1} self.db = self.create_db(files) index.conf['db_default_file'] = None projects.Projects(db_path=self.db, con=self.con, dump_yaml_in_index=True)
def test_get_projects(self): f1 = """ project-templates: default: uri: https://bitbucket.com/%(name)s branches: - master - ocata gitweb: https://bitbucket.com/%(name)s/commit/%%(sha)s projects: Barbican: logo: https://logo.png description: Credentials storage repos: openstack/barbican: template: default openstack/python-barbicanclient: template: default Swift: repos: openstack/swift: template: default openstack/python-swiftclient: template: default branches: - master - 1.0-dev - 2.0-dev """ files = {'f1.yaml': f1} db = self.create_db(files) index.conf['db_default_file'] = None p = projects.Projects(db_path=db) self.assertEqual(len(p.get_projects()['Swift']['repos']), 5) self.assertEqual(len(p.get_projects()['Barbican']['repos']), 4) branches = [ ref['branch'] for ref in p.get_projects()["Swift"]['repos'] if ref['name'] == 'openstack/python-swiftclient'] self.assertListEqual(branches, ['master', '1.0-dev', '2.0-dev'])
def test_projects_validate(self): f1 = """ project-templates: default: uri: https://github.com/%(name)s branches: - master gitweb: https://github.com/openstack/%(name)s/commit/%%(sha)s """ f2 = """ projects: Barbican: repos: openstack/barbican: template: mytemplate """ files = {'f1.yaml': f1, 'f2.yaml': f2} db = self.create_db(files) index.conf['db_default_file'] = None with self.assertRaises(RuntimeError) as exc: projects.Projects(db_path=db, con=self.con, dump_yaml_in_index=True) self.assertIn( "Project ID 'Barbican' Repo ID 'openstack/barbican' " "references an unknown template mytemplate", exc) self.assertEqual(len(exc), 1) f1 = """ project-templates: default: uri: https://github.com/%(name)s branches: - master gitweb: https://github.com/openstack/%(name)s/commit/%%(sha)s releases: - name: "1.0" date: wrong """ files = {'f1.yaml': f1} db = self.create_db(files) index.conf['db_default_file'] = None with self.assertRaises(RuntimeError) as exc: projects.Projects(db_path=db, con=self.con, dump_yaml_in_index=True) self.assertIn( 'Wrong date format wrong defined in template default', exc) self.assertEqual(len(exc), 1) f1 = """ project-templates: default: uri: https://github.com/%(name)s gitweb: https://github.com/openstack/%(name)s/commit/%%(sha)s """ files = {'f1.yaml': f1} db = self.create_db(files) index.conf['db_default_file'] = None with self.assertRaises(RuntimeError) as exc: projects.Projects(db_path=db, con=self.con, dump_yaml_in_index=True) self.assertIn("'branches' is a required property", exc) self.assertEqual(len(exc), 1) f1 = """ projects: Barbican: repos: openstack/barbican: uri: https://github.com/%(name)s template: default """ files = {'f1.yaml': f1} db = self.create_db(files) index.conf['db_default_file'] = None with self.assertRaises(RuntimeError) as exc: projects.Projects(db_path=db, con=self.con, dump_yaml_in_index=True) self.assertIn( "Additional properties are not allowed" " ('uri' was unexpected)", exc) self.assertEqual(len(exc), 1)
def test_exists(self): self.simple_provision() p = projects.Projects(db_path=self.db, con=self.con) self.assertTrue(p.exists('Barbican')) self.assertFalse(p.exists('Swift'))
def test_get_by_id(self): self.simple_provision() p = projects.Projects(db_path=self.db, con=self.con) project = p.get('Barbican') self.assertEqual(project['name'], 'Barbican')
def test_cleaner(self): pi = indexer.RepoIndexer('p1', 'file:///tmp/p1', con=self.con) # This is the initial commits list of a repository we # are going to index repo_commits1 = [ { 'sha': '3597334f2cb10772950c97ddf2f6cc17b184', 'author_date': 1410456005, 'committer_date': 1410456005, 'author_name': 'Nakata Daisuke', 'committer_name': 'Nakata Daisuke', 'author_email': '*****@*****.**', 'committer_email': '*****@*****.**', 'repos': [ 'file:///tmp/p1:p1:master', ], 'line_modifieds': 10, 'commit_msg': 'Add init method', }, ] pi.commits = [rc['sha'] for rc in repo_commits1] pi.set_branch('master') # Start the indexation pi.get_current_commit_indexed() pi.compute_to_index_to_delete() self.init_fake_process_commits_desc_output(pi, repo_commits1) pi.index() repo_commits2 = [ { 'sha': '3597334f2cb10772950c97ddf2f6cc17b185', 'author_date': 1410456005, 'committer_date': 1410456005, 'author_name': 'Nakata Daisuke', 'committer_name': 'Nakata Daisuke', 'author_email': '*****@*****.**', 'committer_email': '*****@*****.**', 'repos': [ 'file:///tmp/p1:p1:devel', 'meta_ref: Fedora'], 'line_modifieds': 10, 'commit_msg': 'Add init method', }, { 'sha': '3597334f2cb10772950c97ddf2f6cc17b186', 'author_date': 1410456005, 'committer_date': 1410456005, 'author_name': 'Nakata Daisuke', 'committer_name': 'Nakata Daisuke', 'author_email': '*****@*****.**', 'committer_email': '*****@*****.**', 'repos': [ 'file:///tmp/p1:p1:devel', 'file:///tmp/p1:p1:master', 'meta_ref: Fedora'], 'line_modifieds': 10, 'commit_msg': 'Add init method', }, ] pi.commits = [rc['sha'] for rc in repo_commits2] pi.set_branch('devel') # Start the indexation pi.get_current_commit_indexed() pi.compute_to_index_to_delete() self.init_fake_process_commits_desc_output(pi, repo_commits2) pi.index() shas = ['3597334f2cb10772950c97ddf2f6cc17b184', '3597334f2cb10772950c97ddf2f6cc17b185', '3597334f2cb10772950c97ddf2f6cc17b186'] pi.tags = [['3597334f2cb10772950c97ddf2f6cc17b184', 'refs/tags/t1'], ['3597334f2cb10772950c97ddf2f6cc17b185', 'refs/tags/t2']] pi.index_tags() self.assertEqual(len(pi.t.get_tags(['file:///tmp/p1:p1'])), 2) # Check 3 commits are indexed self.assertEqual( len([c for c in self.cmts.get_commits_by_id(shas)['docs'] if c['found']]), 3) # Now create the RefsCleaner instance # '3597334f2cb10772950c97ddf2f6cc17b185' will be removed # '3597334f2cb10772950c97ddf2f6cc17b186' will be updated # as the devel branch is no longer referenced with patch.object(index.YAMLBackend, 'load_db'): with patch.object(projects.Projects, 'get_projects') as gp: projects_index = projects.Projects('/tmp/fakepath') gp.return_value = { 'p1': { 'refs': [ {'branch': 'master', 'shortrid': 'file:///tmp/p1:p1', 'fullrid': 'file:///tmp/p1:p1:master', 'uri': 'file:///tmp/p1'} ] } } rc = indexer.RefsCleaner(projects_index, con=self.con) refs_to_clean = rc.find_refs_to_clean() rc.clean(refs_to_clean) # Two commits must be in the db (two was from the master branch) cmts = self.cmts.get_commits_by_id(shas)['docs'] self.assertEqual(len([c for c in cmts if c['found']]), 2) # Verify that remaining commits belong to ref # 'file:///tmp/p1:p1:master' only for cmt in cmts: if not cmt['found']: continue self.assertIn( 'file:///tmp/p1:p1:master', cmt['_source']['repos']) self.assertNotIn( 'file:///tmp/p1:p1:devel', cmt['_source']['repos']) # Here make sure tags are still reference as the base_id still exists self.assertEqual(len(pi.t.get_tags(['file:///tmp/p1:p1'])), 2) # Reinstance a RefsCleaner with no repos with patch.object(index.YAMLBackend, 'load_db'): with patch.object(projects.Projects, 'get_projects') as gp: projects_index = projects.Projects('/tmp/fakepath') gp.return_value = { 'p1': { 'refs': [] } } rc = indexer.RefsCleaner(projects_index, con=self.con) refs_to_clean = rc.find_refs_to_clean() rc.clean(refs_to_clean) # Make sure tags have been deleted self.assertEqual(len(pi.t.get_tags(['file:///tmp/p1:p1'])), 0)
def test_get_projects_raw(self): f1 = """ project-templates: mytemplate: uri: https://bitbucket.com/%(name)s branches: - master gitweb: https://bitbucket.com/%(name)s/commit/%%(sha)s releases: - name: 1.0 date: 2016-12-20 - name: 2.0 date: 2016-12-31 projects: Barbican: repos: openstack/barbican: template: mytemplate openstack/python-barbicanclient: template: mytemplate Swift: repos: openstack/swift: template: default openstack/python-swiftclient: template: default """ default = """ project-templates: default: uri: https://github.com/%(name)s branches: - master - stable/newton - stable/ocata gitweb: https://github.com/openstack/%(name)s/commit/%%(sha)s tags: - openstack parsers: - .*(blueprint) ([^ .]+).* projects: Barbican: repos: openstack/barbican: template: default openstack/python-barbicanclient: template: default Nova: repos: openstack/nova: template: default paths: - tests/ openstack/python-novaclient: template: default """ files = {'f1.yaml': f1, 'default.yaml': default} db = self.create_db(files) index.conf['db_default_file'] = os.path.join(db, 'default.yaml') p = projects.Projects(db_path=db) ret = p.get_projects_raw() expected_ret = { 'Nova': { 'repos': { 'openstack/python-novaclient': { 'tags': ['openstack'], 'branches': ['master', 'stable/newton', 'stable/ocata'], 'parsers': ['.*(blueprint) ([^ .]+).*'], 'gitweb': 'https://github.com/openstack/openstack/' 'python-novaclient/commit/%(sha)s', 'releases': [], 'uri': 'https://github.com/openstack/python-novaclient'}, 'openstack/nova': { 'tags': ['openstack'], 'branches': ['master', 'stable/newton', 'stable/ocata'], 'parsers': ['.*(blueprint) ([^ .]+).*'], 'gitweb': 'https://github.com/openstack/openstack/' 'nova/commit/%(sha)s', 'releases': [], 'paths': ['tests/'], 'uri': 'https://github.com/openstack/nova'}}, }, 'Swift': { 'repos': { 'openstack/swift': { 'tags': ['openstack'], 'branches': ['master', 'stable/newton', 'stable/ocata'], 'parsers': ['.*(blueprint) ([^ .]+).*'], 'gitweb': 'https://github.com/openstack/openstack/' 'swift/commit/%(sha)s', 'releases': [], 'uri': 'https://github.com/openstack/swift'}, 'openstack/python-swiftclient': { 'tags': ['openstack'], 'branches': ['master', 'stable/newton', 'stable/ocata'], 'parsers': ['.*(blueprint) ([^ .]+).*'], 'gitweb': 'https://github.com/openstack/openstack/' 'python-swiftclient/commit/%(sha)s', 'releases': [], 'uri': 'https://github.com/openstack/python-swiftclient'} }, }, 'Barbican': { 'repos': { 'openstack/barbican': { 'branches': ['master'], 'parsers': [], 'gitweb': 'https://bitbucket.com/openstack/' 'barbican/commit/%(sha)s', 'uri': 'https://bitbucket.com/openstack/barbican', 'tags': [], 'releases': [ {'name': 1.0, 'date': 1482192000.0}, {'name': 2.0, 'date': 1483142400.0}]}, 'openstack/python-barbicanclient': { 'branches': ['master'], 'parsers': [], 'gitweb': 'https://bitbucket.com/openstack/' 'python-barbicanclient/commit/%(sha)s', 'uri': 'https://bitbucket.com/openstack/' 'python-barbicanclient', 'tags': [], 'releases': [ {'name': 1.0, 'date': 1482192000.0}, {'name': 2.0, 'date': 1483142400.0}]}}} } self.assertDictEqual(expected_ret, ret)