def test_get_projects(self):
f1 = """
project-templates:
default:
uri: https://bitbucket.com/%(name)s
branches:
- master
- ocata
gitweb: https://bitbucket.com/%(name)s/commit/%%(sha)s
projects:
Barbican:
logo: https://logo.png
description: Credentials storage
repos:
openstack/barbican:
template: default
openstack/python-barbicanclient:
template: default
Swift:
repos:
openstack/swift:
template: default
openstack/python-swiftclient:
template: default
description: The Swift client
forks: 10
watchers: 20
stars: 30
branches:
- master
- 1.0-dev
- 2.0-dev
"""
self.simple_provision(data=f1)
p = projects.Projects(db_path=self.db, con=self.con)
self.assertEqual(len(p.get_projects()['Swift']['refs']), 5)
self.assertEqual(len(p.get_projects()['Barbican']['refs']), 4)
branches = [
ref['branch'] for ref in p.get_projects()["Swift"]['refs']
if ref['name'] == 'openstack/python-swiftclient'
]
self.assertListEqual(branches, ['master', '1.0-dev', '2.0-dev'])
p = projects.Projects(con=self.con)
self.assertEqual(len(p.get_projects()['Swift']['refs']), 5)
ref = [
r for r in p.get_projects()['Swift']['refs']
if r['name'] == 'openstack/python-swiftclient'
][0]
self.assertEqual(ref['description'], 'The Swift client')
self.assertEqual(ref['forks'], 10)
self.assertEqual(ref['watchers'], 20)
self.assertEqual(ref['stars'], 30)
# Check we get only the field name
ret = p.get_projects(source=['name'])
self.assertEqual(len(ret['Swift']), 1)
self.assertIn('name', ret['Swift'])
def test_get_projects_from_references(self):
f1 = """
project-templates:
default:
uri: https://bitbucket.com/%(name)s
branches:
- master
gitweb: https://bitbucket.com/%(name)s/commit/%%(sha)s
projects:
Barbican:
repos:
openstack/barbican:
template: default
Swift:
repos:
openstack/swift:
template: default
Nova:
repos:
openstack/nova:
template: default
"""
self.simple_provision(data=f1)
p = projects.Projects(db_path=self.db, con=self.con)
refs = [('https://bitbucket.com/openstack/barbican:'
'openstack/barbican:master'),
('https://bitbucket.com/openstack/swift:'
'openstack/swift:master')]
ret = p.get_projects_from_references(refs)
self.assertIn('Swift', ret)
self.assertIn('Barbican', ret)
self.assertEqual(len(ret), 2)
def test_get_gitweb_link(self):
self.simple_provision()
p = projects.Projects(db_path=self.db, con=self.con)
link = p.get_gitweb_link(('https://bitbucket.com/openstack/barbican:'
'openstack/barbican:master'))
self.assertEqual(
link, 'https://bitbucket.com/openstack/barbican/commit/%(sha)s')
link = p.get_gitweb_link('notexists')
self.assertEqual(link, '')
def test_get_references_from_tags(self):
f1 = """
project-templates:
default:
uri: https://bitbucket.com/%(name)s
branches:
- master
- ocata
gitweb: https://bitbucket.com/%(name)s/commit/%%(sha)s
tags:
- openstack
- cloud
projects:
Barbican:
repos:
openstack/barbican:
template: default
tags:
- credentials
- server
openstack/python-barbicanclient:
template: default
tags:
- credentials
- client
Swift:
repos:
openstack/swift:
template: default
tags:
- storage
- server
- object-storage
openstack/python-swiftclient:
template: default
tags:
- storage
- client
"""
self.simple_provision(data=f1)
p = projects.Projects(db_path=self.db, con=self.con)
refs = p.get_references_from_tags(['credentials'])
self.assertIn(
{
'fullrid':
('https://bitbucket.com/openstack/python-barbicanclient:'
'openstack/python-barbicanclient:ocata'),
'name':
'openstack/python-barbicanclient',
'branch':
'ocata'
}, refs)
self.assertEqual(len(refs), 4)
refs = p.get_references_from_tags(['credentials', 'object-storage'])
self.assertEqual(len(refs), 6)
def test_get_tags(self):
f1 = """
project-templates:
default:
uri: https://bitbucket.com/%(name)s
branches:
- master
- ocata
gitweb: https://bitbucket.com/%(name)s/commit/%%(sha)s
tags:
- openstack
- cloud
projects:
Barbican:
repos:
openstack/barbican:
template: default
tags:
- credentials
- server
openstack/python-barbicanclient:
template: default
tags:
- credentials
- client
Swift:
repos:
openstack/swift:
template: default
tags:
- storage
- server
openstack/python-swiftclient:
template: default
tags:
- storage
- client
"""
files = {'f1.yaml': f1}
db = self.create_db(files)
index.conf['db_default_file'] = None
p = projects.Projects(db_path=db)
tags = p.get_tags()
self.assertEqual(len(tags['credentials']['repos']), 4)
self.assertEqual(len(tags['storage']['repos']), 4)
self.assertEqual(len(tags.keys()), 6)
for tag in ('openstack', 'cloud', 'client', 'server',
'credentials', 'storage'):
self.assertIn(tag, tags.keys())
def test_get_tags(self):
f1 = """
project-templates:
default:
uri: https://bitbucket.com/%(name)s
branches:
- master
- ocata
gitweb: https://bitbucket.com/%(name)s/commit/%%(sha)s
tags:
- openstack
- cloud
projects:
Barbican:
repos:
openstack/barbican:
template: default
tags:
- credentials
- server
openstack/python-barbicanclient:
template: default
tags:
- credentials
- client
Swift:
repos:
openstack/swift:
template: default
tags:
- storage
- server
openstack/python-swiftclient:
template: default
tags:
- storage
- client
"""
self.simple_provision(data=f1)
p = projects.Projects(db_path=self.db, con=self.con)
tags = p.get_tags()
for tag in ('openstack', 'cloud', 'client', 'server', 'credentials',
'storage'):
self.assertIn(tag, tags)
def simple_provision(self, data=None):
f1 = """
project-templates:
default:
uri: https://bitbucket.com/%(name)s
branches:
- master
gitweb: https://bitbucket.com/%(name)s/commit/%%(sha)s
projects:
Barbican:
repos:
openstack/barbican:
template: default
"""
files = {'f1.yaml': data or f1}
self.db = self.create_db(files)
index.conf['db_default_file'] = None
projects.Projects(db_path=self.db,
con=self.con,
dump_yaml_in_index=True)
def test_get_projects(self):
f1 = """
project-templates:
default:
uri: https://bitbucket.com/%(name)s
branches:
- master
- ocata
gitweb: https://bitbucket.com/%(name)s/commit/%%(sha)s
projects:
Barbican:
logo: https://logo.png
description: Credentials storage
repos:
openstack/barbican:
template: default
openstack/python-barbicanclient:
template: default
Swift:
repos:
openstack/swift:
template: default
openstack/python-swiftclient:
template: default
branches:
- master
- 1.0-dev
- 2.0-dev
"""
files = {'f1.yaml': f1}
db = self.create_db(files)
index.conf['db_default_file'] = None
p = projects.Projects(db_path=db)
self.assertEqual(len(p.get_projects()['Swift']['repos']), 5)
self.assertEqual(len(p.get_projects()['Barbican']['repos']), 4)
branches = [
ref['branch'] for ref in p.get_projects()["Swift"]['repos'] if
ref['name'] == 'openstack/python-swiftclient']
self.assertListEqual(branches, ['master', '1.0-dev', '2.0-dev'])
def test_projects_validate(self):
f1 = """
project-templates:
default:
uri: https://github.com/%(name)s
branches:
- master
gitweb: https://github.com/openstack/%(name)s/commit/%%(sha)s
"""
f2 = """
projects:
Barbican:
repos:
openstack/barbican:
template: mytemplate
"""
files = {'f1.yaml': f1, 'f2.yaml': f2}
db = self.create_db(files)
index.conf['db_default_file'] = None
with self.assertRaises(RuntimeError) as exc:
projects.Projects(db_path=db,
con=self.con,
dump_yaml_in_index=True)
self.assertIn(
"Project ID 'Barbican' Repo ID 'openstack/barbican' "
"references an unknown template mytemplate", exc)
self.assertEqual(len(exc), 1)
f1 = """
project-templates:
default:
uri: https://github.com/%(name)s
branches:
- master
gitweb: https://github.com/openstack/%(name)s/commit/%%(sha)s
releases:
- name: "1.0"
date: wrong
"""
files = {'f1.yaml': f1}
db = self.create_db(files)
index.conf['db_default_file'] = None
with self.assertRaises(RuntimeError) as exc:
projects.Projects(db_path=db,
con=self.con,
dump_yaml_in_index=True)
self.assertIn(
'Wrong date format wrong defined in template default', exc)
self.assertEqual(len(exc), 1)
f1 = """
project-templates:
default:
uri: https://github.com/%(name)s
gitweb: https://github.com/openstack/%(name)s/commit/%%(sha)s
"""
files = {'f1.yaml': f1}
db = self.create_db(files)
index.conf['db_default_file'] = None
with self.assertRaises(RuntimeError) as exc:
projects.Projects(db_path=db,
con=self.con,
dump_yaml_in_index=True)
self.assertIn("'branches' is a required property", exc)
self.assertEqual(len(exc), 1)
f1 = """
projects:
Barbican:
repos:
openstack/barbican:
uri: https://github.com/%(name)s
template: default
"""
files = {'f1.yaml': f1}
db = self.create_db(files)
index.conf['db_default_file'] = None
with self.assertRaises(RuntimeError) as exc:
projects.Projects(db_path=db,
con=self.con,
dump_yaml_in_index=True)
self.assertIn(
"Additional properties are not allowed"
" ('uri' was unexpected)", exc)
self.assertEqual(len(exc), 1)
def test_exists(self):
self.simple_provision()
p = projects.Projects(db_path=self.db, con=self.con)
self.assertTrue(p.exists('Barbican'))
self.assertFalse(p.exists('Swift'))
def test_get_by_id(self):
self.simple_provision()
p = projects.Projects(db_path=self.db, con=self.con)
project = p.get('Barbican')
self.assertEqual(project['name'], 'Barbican')
def test_cleaner(self):
pi = indexer.RepoIndexer('p1', 'file:///tmp/p1',
con=self.con)
# This is the initial commits list of a repository we
# are going to index
repo_commits1 = [
{
'sha': '3597334f2cb10772950c97ddf2f6cc17b184',
'author_date': 1410456005,
'committer_date': 1410456005,
'author_name': 'Nakata Daisuke',
'committer_name': 'Nakata Daisuke',
'author_email': '*****@*****.**',
'committer_email': '*****@*****.**',
'repos': [
'file:///tmp/p1:p1:master', ],
'line_modifieds': 10,
'commit_msg': 'Add init method',
},
]
pi.commits = [rc['sha'] for rc in repo_commits1]
pi.set_branch('master')
# Start the indexation
pi.get_current_commit_indexed()
pi.compute_to_index_to_delete()
self.init_fake_process_commits_desc_output(pi, repo_commits1)
pi.index()
repo_commits2 = [
{
'sha': '3597334f2cb10772950c97ddf2f6cc17b185',
'author_date': 1410456005,
'committer_date': 1410456005,
'author_name': 'Nakata Daisuke',
'committer_name': 'Nakata Daisuke',
'author_email': '*****@*****.**',
'committer_email': '*****@*****.**',
'repos': [
'file:///tmp/p1:p1:devel', 'meta_ref: Fedora'],
'line_modifieds': 10,
'commit_msg': 'Add init method',
},
{
'sha': '3597334f2cb10772950c97ddf2f6cc17b186',
'author_date': 1410456005,
'committer_date': 1410456005,
'author_name': 'Nakata Daisuke',
'committer_name': 'Nakata Daisuke',
'author_email': '*****@*****.**',
'committer_email': '*****@*****.**',
'repos': [
'file:///tmp/p1:p1:devel',
'file:///tmp/p1:p1:master',
'meta_ref: Fedora'],
'line_modifieds': 10,
'commit_msg': 'Add init method',
},
]
pi.commits = [rc['sha'] for rc in repo_commits2]
pi.set_branch('devel')
# Start the indexation
pi.get_current_commit_indexed()
pi.compute_to_index_to_delete()
self.init_fake_process_commits_desc_output(pi, repo_commits2)
pi.index()
shas = ['3597334f2cb10772950c97ddf2f6cc17b184',
'3597334f2cb10772950c97ddf2f6cc17b185',
'3597334f2cb10772950c97ddf2f6cc17b186']
pi.tags = [['3597334f2cb10772950c97ddf2f6cc17b184', 'refs/tags/t1'],
['3597334f2cb10772950c97ddf2f6cc17b185', 'refs/tags/t2']]
pi.index_tags()
self.assertEqual(len(pi.t.get_tags(['file:///tmp/p1:p1'])), 2)
# Check 3 commits are indexed
self.assertEqual(
len([c for c in self.cmts.get_commits_by_id(shas)['docs']
if c['found']]), 3)
# Now create the RefsCleaner instance
# '3597334f2cb10772950c97ddf2f6cc17b185' will be removed
# '3597334f2cb10772950c97ddf2f6cc17b186' will be updated
# as the devel branch is no longer referenced
with patch.object(index.YAMLBackend, 'load_db'):
with patch.object(projects.Projects, 'get_projects') as gp:
projects_index = projects.Projects('/tmp/fakepath')
gp.return_value = {
'p1': {
'refs': [
{'branch': 'master',
'shortrid': 'file:///tmp/p1:p1',
'fullrid': 'file:///tmp/p1:p1:master',
'uri': 'file:///tmp/p1'}
]
}
}
rc = indexer.RefsCleaner(projects_index, con=self.con)
refs_to_clean = rc.find_refs_to_clean()
rc.clean(refs_to_clean)
# Two commits must be in the db (two was from the master branch)
cmts = self.cmts.get_commits_by_id(shas)['docs']
self.assertEqual(len([c for c in cmts if c['found']]), 2)
# Verify that remaining commits belong to ref
# 'file:///tmp/p1:p1:master' only
for cmt in cmts:
if not cmt['found']:
continue
self.assertIn(
'file:///tmp/p1:p1:master', cmt['_source']['repos'])
self.assertNotIn(
'file:///tmp/p1:p1:devel', cmt['_source']['repos'])
# Here make sure tags are still reference as the base_id still exists
self.assertEqual(len(pi.t.get_tags(['file:///tmp/p1:p1'])), 2)
# Reinstance a RefsCleaner with no repos
with patch.object(index.YAMLBackend, 'load_db'):
with patch.object(projects.Projects, 'get_projects') as gp:
projects_index = projects.Projects('/tmp/fakepath')
gp.return_value = {
'p1': {
'refs': []
}
}
rc = indexer.RefsCleaner(projects_index, con=self.con)
refs_to_clean = rc.find_refs_to_clean()
rc.clean(refs_to_clean)
# Make sure tags have been deleted
self.assertEqual(len(pi.t.get_tags(['file:///tmp/p1:p1'])), 0)
def test_get_projects_raw(self):
f1 = """
project-templates:
mytemplate:
uri: https://bitbucket.com/%(name)s
branches:
- master
gitweb: https://bitbucket.com/%(name)s/commit/%%(sha)s
releases:
- name: 1.0
date: 2016-12-20
- name: 2.0
date: 2016-12-31
projects:
Barbican:
repos:
openstack/barbican:
template: mytemplate
openstack/python-barbicanclient:
template: mytemplate
Swift:
repos:
openstack/swift:
template: default
openstack/python-swiftclient:
template: default
"""
default = """
project-templates:
default:
uri: https://github.com/%(name)s
branches:
- master
- stable/newton
- stable/ocata
gitweb: https://github.com/openstack/%(name)s/commit/%%(sha)s
tags:
- openstack
parsers:
- .*(blueprint) ([^ .]+).*
projects:
Barbican:
repos:
openstack/barbican:
template: default
openstack/python-barbicanclient:
template: default
Nova:
repos:
openstack/nova:
template: default
paths:
- tests/
openstack/python-novaclient:
template: default
"""
files = {'f1.yaml': f1, 'default.yaml': default}
db = self.create_db(files)
index.conf['db_default_file'] = os.path.join(db,
'default.yaml')
p = projects.Projects(db_path=db)
ret = p.get_projects_raw()
expected_ret = {
'Nova': {
'repos': {
'openstack/python-novaclient': {
'tags': ['openstack'],
'branches': ['master', 'stable/newton',
'stable/ocata'],
'parsers': ['.*(blueprint) ([^ .]+).*'],
'gitweb': 'https://github.com/openstack/openstack/'
'python-novaclient/commit/%(sha)s',
'releases': [],
'uri':
'https://github.com/openstack/python-novaclient'},
'openstack/nova': {
'tags': ['openstack'],
'branches': ['master', 'stable/newton',
'stable/ocata'],
'parsers': ['.*(blueprint) ([^ .]+).*'],
'gitweb': 'https://github.com/openstack/openstack/'
'nova/commit/%(sha)s',
'releases': [],
'paths': ['tests/'],
'uri': 'https://github.com/openstack/nova'}},
},
'Swift': {
'repos': {
'openstack/swift': {
'tags': ['openstack'],
'branches': ['master', 'stable/newton',
'stable/ocata'],
'parsers': ['.*(blueprint) ([^ .]+).*'],
'gitweb': 'https://github.com/openstack/openstack/'
'swift/commit/%(sha)s',
'releases': [],
'uri': 'https://github.com/openstack/swift'},
'openstack/python-swiftclient': {
'tags': ['openstack'],
'branches': ['master', 'stable/newton',
'stable/ocata'],
'parsers': ['.*(blueprint) ([^ .]+).*'],
'gitweb': 'https://github.com/openstack/openstack/'
'python-swiftclient/commit/%(sha)s',
'releases': [],
'uri':
'https://github.com/openstack/python-swiftclient'}
},
},
'Barbican': {
'repos': {
'openstack/barbican': {
'branches': ['master'],
'parsers': [],
'gitweb': 'https://bitbucket.com/openstack/'
'barbican/commit/%(sha)s',
'uri': 'https://bitbucket.com/openstack/barbican',
'tags': [],
'releases': [
{'name': 1.0, 'date': 1482192000.0},
{'name': 2.0, 'date': 1483142400.0}]},
'openstack/python-barbicanclient': {
'branches': ['master'],
'parsers': [],
'gitweb': 'https://bitbucket.com/openstack/'
'python-barbicanclient/commit/%(sha)s',
'uri': 'https://bitbucket.com/openstack/'
'python-barbicanclient',
'tags': [],
'releases': [
{'name': 1.0, 'date': 1482192000.0},
{'name': 2.0, 'date': 1483142400.0}]}}}
}
self.assertDictEqual(expected_ret, ret)
