def runTest(self):
fn = 'metadata_1.yml'
fn = os.path.join(os.path.dirname(__file__), fn)
with open(fn, 'rb') as f:
data = f.read()
pdata = BotMetadataParser.parse_yaml(data)
def test_keywords(self):
LABELS_SUPPORT_PROPAGATION = """
macros:
module_utils: lib/ansible/module_utils
files:
$module_utils/network/:
support: network
labels: networking
$module_utils/network/fw:
labels: firewall
$module_utils/network/fw/sub:
support: core
labels: [fwsub]
$module_utils/network/fw/sub/childA:
$module_utils/network/fw/sub/childB:
support: another_level
labels: labelB
$module_utils/network/iwfu.py:
support: community
labels: firewall
$module_utils/other:
labels: thing
"""
data = BotMetadataParser.parse_yaml(LABELS_SUPPORT_PROPAGATION)
self._test(data)
def parse_metadata(self):
if self.botmetafile is not None:
with open(self.botmetafile, 'rb') as f:
rdata = f.read()
else:
fp = '.github/BOTMETA.yml'
rdata = self.get_file_content(fp)
if rdata:
self.botmeta = BotMetadataParser.parse_yaml(rdata)
else:
self.botmeta = {}
# reshape meta into old format
self.CMAP = {}
for k, v in self.botmeta.get('files', {}).items():
if not v:
continue
if 'keywords' not in v:
continue
for keyword in v['keywords']:
if keyword not in self.CMAP:
self.CMAP[keyword] = []
if k not in self.CMAP[keyword]:
self.CMAP[keyword].append(k)
# update the data
self.get_files()
self.get_filemap()
def runTest(self):
data = BotMetadataParser.parse_yaml(EXAMPLE1)
assert 'macros' in data
assert 'files' in data
assert 'lib/ansible/cli/galaxy/' in data['files']
assert 'lib/ansible/cli/vault.py' in data['files']
assert 'lib/ansible/parsing/vault/' in data['files']
assert 'lib/ansible/foobar/' in data['files']
self.assertEqual(data['files']['lib/ansible/foobar/']['labels'],
['ansible', 'bar', 'foo', 'foobar', 'lib'])
self.assertEqual(
data['files']['lib/ansible/cli/vault.py']['maintainers'],
['larry', 'curly', 'moe', 'jeff'])
# double-macro
assert 'lib/ansible/modules/x/y' in data['files']
assert 'maintainers' in data['files']['lib/ansible/modules/x/y']
self.assertEqual(
data['files']['lib/ansible/modules/x/y']['maintainers'],
['steven'])
assert 'team_oneline' in data['macros']
assert isinstance(data['macros']['team_oneline'], list)
self.assertEqual(data['macros']['team_oneline'],
['one', 'line', 'at', 'a', 'time'])
def parse_metadata(self):
if self.botmetafile is not None:
with open(self.botmetafile, 'rb') as f:
rdata = f.read()
else:
fp = u'.github/BOTMETA.yml'
rdata = self.get_file_content(fp)
if rdata:
self.botmeta = BotMetadataParser.parse_yaml(rdata)
else:
self.botmeta = {}
# reshape meta into old format
self.CMAP = {}
for k, v in self.botmeta.get(u'files', {}).items():
if not v:
continue
if u'keywords' not in v:
continue
for keyword in v[u'keywords']:
if keyword not in self.CMAP:
self.CMAP[keyword] = []
if k not in self.CMAP[keyword]:
self.CMAP[keyword].append(k)
# update the data
self.get_files()
self.get_filemap()
def runTest(self):
fn = 'metadata_1.yml'
fn = os.path.join(os.path.dirname(__file__), fn)
with open(fn, 'rb') as f:
data = f.read()
pdata = BotMetadataParser.parse_yaml(data)
def test_keywords_order(self):
"""Check that:
- labels are inherited
- support keyword is inherited when not set
Use ruamel in order to check that order does't count
"""
LABELS_SUPPORT_PROPAGATION = """
macros:
module_utils: lib/ansible/module_utils
files:
$module_utils/network/fw/sub/childA:
$module_utils/network/fw/sub/childB:
support: another_level
labels: labelB
$module_utils/network/fw/sub:
support: core
labels: [fwsub]
$module_utils/other:
labels: thing
$module_utils/network/iwfu.py:
support: community
labels: firewall
$module_utils/network/:
support: network
labels: networking
$module_utils/network/fw:
labels: firewall
"""
data = BotMetadataParser.parse_yaml(LABELS_SUPPORT_PROPAGATION)
self.assertIsInstance(data, ruamel.yaml.comments.CommentedMap) # ensure mock is effective
self._test(data)
def runTest(self):
data = BotMetadataParser.parse_yaml(EXAMPLE1)
assert u'macros' in data
assert u'files' in data
assert u'lib/ansible/cli/galaxy' in data[u'files']
assert u'lib/ansible/cli/vault.py' in data[u'files']
assert u'lib/ansible/parsing/vault' in data[u'files']
assert u'lib/ansible/foobar' in data[u'files']
self.assertEqual(data[u'files'][u'lib/ansible/foobar'][u'labels'],
[u'ansible', u'bar', u'foo', u'foobar', u'lib'])
self.assertEqual(
data[u'files'][u'lib/ansible/cli/vault.py'][u'maintainers'],
[u'larry', u'curly', u'moe', u'jeff'],
)
# double-macro
assert u'lib/ansible/modules/x/y' in data[u'files']
assert u'maintainers' in data[u'files'][u'lib/ansible/modules/x/y']
self.assertEqual(
data[u'files'][u'lib/ansible/modules/x/y'][u'maintainers'],
[u'steven'])
assert u'team_oneline' in data[u'macros']
assert isinstance(data[u'macros'][u'team_oneline'], list)
self.assertEqual(data[u'macros'][u'team_oneline'],
[u'one', u'line', u'at', u'a', u'time'])
self.assertEqual(dict, type(data[u'files'][u'packaging']))
def load_meta(self):
if self.botmetafile is not None:
with open(self.botmetafile, 'rb') as f:
rdata = f.read()
else:
fp = '.github/BOTMETA.yml'
rdata = self.gitrepo.get_file_content(fp)
self.BOTMETA = BotMetadataParser.parse_yaml(rdata)
def parse_metadata(self):
fp = '.github/BOTMETA.yml'
rdata = self.get_file_content(fp)
self.botmeta = BotMetadataParser.parse_yaml(rdata)
# load the modules
logging.info('loading modules')
self.get_ansible_modules()
def parse_metadata(self):
if self.botmetafile is not None:
with open(self.botmetafile, 'rb') as f:
rdata = f.read()
else:
fp = u'.github/BOTMETA.yml'
rdata = self.get_file_content(fp)
self.botmeta = BotMetadataParser.parse_yaml(rdata)
# load the modules
logging.info(u'loading modules')
self.get_ansible_modules()
def parse_metadata(self):
if self.botmetafile is not None:
with open(self.botmetafile, 'rb') as f:
rdata = f.read()
else:
fp = u'.github/BOTMETA.yml'
rdata = self.get_file_content(fp)
self.botmeta = BotMetadataParser.parse_yaml(rdata)
# load the modules
logging.info(u'loading modules')
self.get_ansible_modules()
def parse_metadata(self):
fp = '.github/BOTMETA.yml'
rdata = self.get_file_content(fp)
self.botmeta = BotMetadataParser.parse_yaml(rdata)
# reshape data to the old format
self.maintainers = {}
for k, v in self.botmeta['files'].items():
fp = k.replace('lib/ansible/modules/', '')
if isinstance(v, dict):
self.maintainers[fp] = v.get('maintainers', [])
else:
self.maintainers[fp] = []
# load the modules
logging.info('loading modules')
self.get_ansible_modules()
def runTest(self):
data = BotMetadataParser.parse_yaml(EXAMPLE_ANCHORS)
# shortcuts
topdir = u'lib/ansible/modules/topdir'
dfile = u'docs/foo/bar'
mfile = u'lib/ansible/modules/topdir/topfile'
# labels should be automatic from the path(s)
assert u'docs' in data[u'files'][dfile][u'labels']
# children should inherit from their parent anchors
assert u'topdir' in data[u'files'][dfile][u'labels']
assert u'topdir' in data[u'files'][dfile][u'labels']
# we do not want pointers merging all data into the anchor
assert u'docs' not in data[u'files'][topdir][u'labels']
assert u'docs' not in data[u'files'][mfile][u'labels']
def runTest(self):
data = BotMetadataParser.parse_yaml(EXAMPLE_ANCHORS)
# shortcuts
topdir = 'lib/ansible/modules/topdir'
dfile = 'docs/foo/bar'
mfile = 'lib/ansible/modules/topdir/topfile'
# labels should be automatic from the path(s)
assert 'docs' in data['files'][dfile]['labels']
# children should inherit from their parent anchors
assert 'topdir' in data['files'][dfile]['labels']
assert 'topdir' in data['files'][dfile]['labels']
# we do not want pointers merging all data into the anchor
assert 'docs' not in data['files'][topdir]['labels']
assert 'docs' not in data['files'][mfile]['labels']
def parse_metadata(self):
fp = '.github/BOTMETA.yml'
rdata = self.get_file_content(fp)
self.botmeta = BotMetadataParser.parse_yaml(rdata)
# reshape meta into old format
self.CMAP = {}
for k,v in self.botmeta['files'].items():
if not v:
continue
if 'keywords' not in v:
continue
for keyword in v['keywords']:
if keyword not in self.CMAP:
self.CMAP[keyword] = []
if k not in self.CMAP[keyword]:
self.CMAP[keyword].append(k)
# update the data
self.get_files()
self.get_filemap()
def runTest(self):
data = BotMetadataParser.parse_yaml(EXAMPLE1)
assert u'macros' in data
assert u'files' in data
assert u'lib/ansible/cli/galaxy' in data[u'files']
assert u'lib/ansible/cli/vault.py' in data[u'files']
assert u'lib/ansible/parsing/vault' in data[u'files']
assert u'lib/ansible/foobar' in data[u'files']
self.assertEqual(
data[u'files'][u'lib/ansible/foobar'][u'labels'],
[u'ansible', u'bar', u'foo', u'foobar', u'lib']
)
self.assertEqual(
data[u'files'][u'lib/ansible/cli/vault.py'][u'maintainers'],
[u'larry', u'curly', u'moe', u'jeff'],
)
# double-macro
assert u'lib/ansible/modules/x/y' in data[u'files']
assert u'maintainers' in data[u'files'][u'lib/ansible/modules/x/y']
self.assertEqual(
data[u'files'][u'lib/ansible/modules/x/y'][u'maintainers'],
[u'steven']
)
assert u'team_oneline' in data[u'macros']
assert isinstance(data[u'macros'][u'team_oneline'], list)
self.assertEqual(
data[u'macros'][u'team_oneline'],
[u'one', u'line', u'at', u'a', u'time']
)
self.assertEqual(dict, type(data[u'files'][u'packaging']))
def test_supported_by_order(self):
"""Check that:
- supported_by keyword is inherited when not set
Use ruamel in order to check that order does't count
"""
LABELS_SUPPORTED_BY_PROPAGATION = """
macros:
module_utils: lib/ansible/module_utils
files:
$module_utils/network/fw/sub/childA:
$module_utils/network/fw/sub/childB:
supported_by: another_level
labels: labelB
$module_utils/network/fw/sub:
supported_by: core
labels: [fwsub]
$module_utils/other:
labels: thing
$module_utils/network/iwfu.py:
supported_by: community
labels: firewall
$module_utils/network/:
supported_by: network
labels: networking
$module_utils/network/fw:
labels: firewall
"""
data = BotMetadataParser.parse_yaml(LABELS_SUPPORTED_BY_PROPAGATION)
self.assertIsInstance(data, ruamel.yaml.comments.CommentedMap) # ensure mock is effective
#### 'supported_by' key
self.assertEqual(
data[u'files'][u'lib/ansible/module_utils/network'][u'supported_by'], [u'network']
)
# subpath: supported_by key is inherited
self.assertEqual(
data[u'files'][u'lib/ansible/module_utils/network/fw'][u'supported_by'], [u'network']
)
# subpath: supported_by key is overridden
self.assertEqual(
data[u'files'][u'lib/ansible/module_utils/network/iwfu.py'][u'supported_by'], [u'community']
)
# subpath: supported_by key is overridden
self.assertEqual(
data[u'files'][u'lib/ansible/module_utils/network/fw/sub'][u'supported_by'], [u'core']
)
# subpath: supported_by key is inherited
self.assertEqual(
data[u'files'][u'lib/ansible/module_utils/network/fw/sub/childA'][u'supported_by'], [u'core']
)
# subpath: supported_by key is overridden
self.assertEqual(
data[u'files'][u'lib/ansible/module_utils/network/fw/sub/childB'][u'supported_by'], [u'another_level']
)
# default value for supported_by isn't set by botmeta
self.assertNotIn(u'supported_by', data[u'files'][u'lib/ansible/module_utils/other'])
def test_keywords(self):
LABELS_SUPPORT_PROPAGATION = """
macros:
module_utils: lib/ansible/module_utils
files:
$module_utils/network/:
support: network
labels: networking
$module_utils/network/fw:
labels: firewall
$module_utils/network/fw/sub:
support: core
labels: [fwsub]
$module_utils/network/fw/sub/childA:
$module_utils/network/fw/sub/childB:
support: another_level
labels: labelB
$module_utils/network/iwfu.py:
support: community
labels: firewall
$module_utils/other:
labels: thing
"""
data = BotMetadataParser.parse_yaml(LABELS_SUPPORT_PROPAGATION)
assert u'macros' in data
assert u'files' in data
self.assertEqual(set([u'lib/ansible/module_utils/network',
u'lib/ansible/module_utils/network/fw',
u'lib/ansible/module_utils/network/fw/sub',
u'lib/ansible/module_utils/network/fw/sub/childA',
u'lib/ansible/module_utils/network/fw/sub/childB',
u'lib/ansible/module_utils/network/iwfu.py',
u'lib/ansible/module_utils/other']),
set(data[u'files'].keys()))
#### 'labels' key
self.assertEqual(
set(data[u'files'][u'lib/ansible/module_utils/network'][u'labels']),
set([u'lib', u'ansible', u'module_utils', u'network', u'networking'])
)
self.assertEqual(
set(data[u'files'][u'lib/ansible/module_utils/network/fw'][u'labels']),
set([u'lib', u'ansible', u'module_utils', u'network', u'networking', u'fw', u'firewall'])
)
self.assertEqual(
set(data[u'files'][u'lib/ansible/module_utils/network/fw/sub'][u'labels']),
set([u'lib', u'ansible', u'module_utils', u'network', u'networking', u'fw', u'firewall', u'sub', u'fwsub'])
)
self.assertEqual(
set(data[u'files'][u'lib/ansible/module_utils/network/fw/sub/childA'][u'labels']),
set([u'lib', u'ansible', u'module_utils', u'network', u'networking', u'fw', u'firewall', u'sub', u'fwsub', u'childA'])
)
self.assertEqual(
set(data[u'files'][u'lib/ansible/module_utils/network/fw/sub/childB'][u'labels']),
set([u'lib', u'ansible', u'module_utils', u'network', u'networking', u'fw', u'firewall', u'sub', u'fwsub', u'childB', u'labelB'])
)
self.assertEqual(
set(data[u'files'][u'lib/ansible/module_utils/network/iwfu.py'][u'labels']),
set([u'lib', u'ansible', u'module_utils', u'network', u'networking', u'iwfu', u'firewall'])
)
#### 'support' key
self.assertEqual(
data[u'files'][u'lib/ansible/module_utils/network'][u'support'], [u'network']
)
# subpath: support key is inherited
self.assertEqual(
data[u'files'][u'lib/ansible/module_utils/network/fw'][u'support'], [u'network']
)
# subpath: support key is overridden
self.assertEqual(
data[u'files'][u'lib/ansible/module_utils/network/iwfu.py'][u'support'], [u'community']
)
# subpath: support key is overridden
self.assertEqual(
data[u'files'][u'lib/ansible/module_utils/network/fw/sub'][u'support'], [u'core']
)
# subpath: support key is inherited
self.assertEqual(
data[u'files'][u'lib/ansible/module_utils/network/fw/sub/childA'][u'support'], [u'core']
)
# subpath: support key is overridden
self.assertEqual(
data[u'files'][u'lib/ansible/module_utils/network/fw/sub/childB'][u'support'], [u'another_level']
)
# default value for support isn't set by botmeta
self.assertNotIn(u'support', data[u'files'][u'lib/ansible/module_utils/other'])
def main():
tocheck = [
#32226,
#30361,
#31006,
#58674,
#63611,
#64320,
#66891,
#68784,
69010,
]
redirect = set()
noredirect = set()
nometa = set()
cachedir = '/home/jtanner/.ansibullbot/cache'
gitrepo = GitRepoWrapper(cachedir=cachedir,
repo='https://github.com/ansible/ansible',
commit=None,
rebase=False)
rdata = gitrepo.get_file_content(u'.github/BOTMETA.yml')
botmeta = BotMetadataParser.parse_yaml(rdata)
cm = AnsibleComponentMatcher(cachedir=cachedir,
gitrepo=gitrepo,
botmeta=botmeta,
botmetafile=None,
email_cache=None,
usecache=True,
use_galaxy=True)
'''
mr = parse_match_results()
for issue in sorted(mr.keys(), key=lambda x: int(x.split('/')[-1]), reverse=True):
print(issue)
number = int(issue.split('/')[-1])
#if number != 68709:
# continue
print(number)
mfile = os.path.join('~/.ansibullbot/cache/ansible/ansible/issues/%s' % number, 'meta.json')
mfile = os.path.expanduser(mfile)
if os.path.exists(mfile):
with open(mfile, 'r') as f:
imeta = json.loads(f.read())
else:
nometa.add(issue)
imeta = {}
if imeta:
iw = MockIssueWrapper(issue, meta=imeta)
cfacts = get_collection_facts(iw, cm, imeta)
#pprint(cfacts)
if cfacts.get('needs_collection_redirect') == True:
redirect.add(issue)
else:
noredirect.add(issue)
#if not imeta['is_backport']:
# import epdb; epdb.st()
'''
mmap = {}
#gmatches = cm.search_ecosystem('contrib/inventory/ec2.py')
#import epdb; epdb.st()
mfiles = get_issues()
for mfile in mfiles:
with open(mfile, 'r') as f:
imeta = json.loads(f.read())
print(imeta['html_url'])
number = int(imeta['html_url'].split('/')[-1])
if number not in tocheck:
continue
newmeta = copy.deepcopy(imeta)
iw = MockIssueWrapper(imeta['html_url'], meta=newmeta, gitrepo=gitrepo)
#cmatches = cm.match_components(iw.title, iw.body, iw.component)
cmmeta = get_component_match_facts(iw, cm, [])
newmeta.update(cmmeta)
cfmeta = get_collection_facts(iw, cm, newmeta)
# check api deltas ...
#cm1 = cm.match(iw)
#cm2 = cm.match_components(iw.title, iw.body, iw.component, files=iw.files)
#import epdb; epdb.st()
print('component: %s' % iw.component)
print(cmmeta['component_filenames'])
#pprint(cfmeta)
cf2vals = [x for x in list(cfmeta['collection_filemap'].values()) if x]
cf1vals = [x for x in list(imeta['collection_filemap'].values()) if x]
'''
if cf1vals or cf2vals:
pprint(cf1vals)
pprint(cf2vals)
#import epdb; epdb.st()
'''
'''
if cf2vals != cf1vals:
pprint(cf1vals)
pprint(cf2vals)
import epdb; epdb.st()
'''
pprint(cfmeta)
import epdb
epdb.st()
print('# %s total issues|PRs without meta' % len(list(nometa)))
print('# %s total issues|PRs not redirected to collections' %
len(list(noredirect)))
print('# %s total issues|PRs redirected to collections' %
len(list(redirect)))
import epdb
epdb.st()
def test_keywords(self):
LABELS_SUPPORT_PROPAGATION = """
macros:
module_utils: lib/ansible/module_utils
files:
$module_utils/network/:
support: network
labels: networking
$module_utils/network/fw:
labels: firewall
$module_utils/network/fw/sub:
support: core
labels: [fwsub]
$module_utils/network/fw/sub/childA:
$module_utils/network/fw/sub/childB:
support: another_level
labels: labelB
$module_utils/network/iwfu.py:
support: community
labels: firewall
$module_utils/other:
labels: thing
"""
data = BotMetadataParser.parse_yaml(LABELS_SUPPORT_PROPAGATION)
assert 'macros' in data
assert 'files' in data
self.assertEqual(
{
'lib/ansible/module_utils/network',
'lib/ansible/module_utils/network/fw',
'lib/ansible/module_utils/network/fw/sub',
'lib/ansible/module_utils/network/fw/sub/childA',
'lib/ansible/module_utils/network/fw/sub/childB',
'lib/ansible/module_utils/network/iwfu.py',
'lib/ansible/module_utils/other'
}, set(data['files'].keys()))
#### 'labels' key
self.assertEqual(
set(data['files']['lib/ansible/module_utils/network']['labels']),
{'lib', 'ansible', 'module_utils', 'network', 'networking'})
self.assertEqual(
set(data['files']['lib/ansible/module_utils/network/fw']
['labels']), {
'lib', 'ansible', 'module_utils', 'network', 'networking',
'fw', 'firewall'
})
self.assertEqual(
set(data['files']['lib/ansible/module_utils/network/fw/sub']
['labels']), {
'lib', 'ansible', 'module_utils', 'network', 'networking',
'fw', 'firewall', 'sub', 'fwsub'
})
self.assertEqual(
set(data['files']['lib/ansible/module_utils/network/fw/sub/childA']
['labels']), {
'lib', 'ansible', 'module_utils', 'network', 'networking',
'fw', 'firewall', 'sub', 'fwsub', 'childA'
})
self.assertEqual(
set(data['files']['lib/ansible/module_utils/network/fw/sub/childB']
['labels']), {
'lib', 'ansible', 'module_utils', 'network', 'networking',
'fw', 'firewall', 'sub', 'fwsub', 'childB', 'labelB'
})
self.assertEqual(
set(data['files']['lib/ansible/module_utils/network/iwfu.py']
['labels']), {
'lib', 'ansible', 'module_utils', 'network', 'networking',
'iwfu', 'firewall'
})
#### 'support' key
self.assertEqual(
data['files']['lib/ansible/module_utils/network']['support'],
['network'])
# subpath: support key is inherited
self.assertEqual(
data['files']['lib/ansible/module_utils/network/fw']['support'],
['network'])
# subpath: support key is overridden
self.assertEqual(
data['files']['lib/ansible/module_utils/network/iwfu.py']
['support'], ['community'])
# subpath: support key is overridden
self.assertEqual(
data['files']['lib/ansible/module_utils/network/fw/sub']
['support'], ['core'])
# subpath: support key is inherited
self.assertEqual(
data['files']['lib/ansible/module_utils/network/fw/sub/childA']
['support'], ['core'])
# subpath: support key is overridden
self.assertEqual(
data['files']['lib/ansible/module_utils/network/fw/sub/childB']
['support'], ['another_level'])
# default value for support isn't set by botmeta
self.assertNotIn('support',
data['files']['lib/ansible/module_utils/other'])