def test_owner_pr_submitter_is_maintainer_new_module(self):
"""
Submitter is a maintainer: pull request adds a new module: ensure owner_pr is False
"""
botmeta_files = {u'lib/ansible/modules/foo/bar.py': {u'maintainers': [u'ElsA', u'mscherer']}}
CM = ComponentMatcherMock()
CM.expected_results = [
{
u'repo_filename': u'lib/ansible/modules/foo/bar.py',
u'labels': [],
u'support': None,
u'maintainers': [u'ElsA', u'mscherer'],
u'notify': [u'ElsA', u'mscherer'],
u'ignore': [],
}
]
meta = self.meta.copy()
datafile = u'tests/fixtures/shipit/0_issue.yml'
statusfile = u'tests/fixtures/shipit/0_prstatus.json'
with get_issue(datafile, statusfile) as iw:
iw.pr_files = [MockFile(u'lib/ansible/modules/foo/bar.py')]
iw.gitrepo = GitRepoWrapperMock()
meta.update(get_component_match_facts(iw, CM, []))
facts = get_shipit_facts(iw, meta, botmeta_files, core_team=[u'bcoca'], botnames=[u'ansibot'])
self.assertEqual(iw.submitter, u'mscherer')
self.assertFalse(facts[u'owner_pr'])
def test_owner_pr_submitter_is_not_maintainer_of_all_updated_files(self):
"""
PR updates 2 files below module_utils, submitter is a maintainer from only one: ensure owner_pr isn't set
"""
BOTMETA = u"""
---
macros:
modules: lib/ansible/modules
module_utils: lib/ansible/module_utils
files:
$module_utils/foo/bar.py:
maintainers: ElsA Oliver
$module_utils/baz/bar.py:
maintainers: TiTi ZaZa
"""
module_indexer = create_indexer(textwrap.dedent(BOTMETA), {})
self.assertEqual(len(module_indexer.modules), 1) # ensure only fake data are loaded
self.assertEqual(sorted(module_indexer.botmeta[u'files'][u'lib/ansible/module_utils/foo/bar.py'][u'maintainers']), [u'ElsA', u'Oliver'])
self.assertEqual(sorted(module_indexer.botmeta[u'files'][u'lib/ansible/module_utils/baz/bar.py'][u'maintainers']), [u'TiTi', u'ZaZa'])
CM = ComponentMatcherMock()
CM.expected_results = [
{
u'repo_filename': u'lib/ansible/module_utils/foo/bar.py',
u'labels': [],
u'support': None,
u'maintainers': [u'ElsA', u'Oliver'],
u'notify': [u'ElsA', u'Oliver'],
u'ignore': [],
},
{
u'repo_filename': u'lib/ansible/modules/baz/bar.py',
u'labels': [],
u'support': None,
u'maintainers': [u'TiTi', u'ZaZa'],
u'notify': [u'TiTi', u'ZaZa'],
u'ignore': [],
}
]
issue = IssueMock(u'/dev/null')
issue.user.login = u'ElsA'
issue.html_url = u'https://github.com/ansible/ansible/pull/123'
iw = IssueWrapper(cachedir="", issue=issue)
iw.pr_files = [
MockFile(u'lib/ansible/module_utils/foo/bar.py'),
MockFile(u'lib/ansible/module_utils/baz/bar.py')
]
iw.file_indexer = FileIndexerMock()
iw.repo = MockRepo(repo_path='ansible/ansible')
meta = self.meta.copy()
iw._commits = []
meta.update(get_component_match_facts(iw, CM, []))
facts = get_shipit_facts(iw, meta, module_indexer, core_team=[u'bcoca', u'mscherer'], botnames=[u'ansibot'])
self.assertEqual(iw.submitter, u'ElsA')
self.assertFalse(facts[u'owner_pr'])
def test_owner_pr_submitter_is_maintainer_one_module_utils_file_updated(self):
"""
Submitter is a maintainer: ensure owner_pr is set (only one file below module_utils updated)
"""
botmeta_files = {'lib/ansible/module_utils/foo/bar.py': {'maintainers': ['ElsA', 'Oliver']}}
datafile = u'tests/fixtures/shipit/2_issue.yml'
statusfile = u'tests/fixtures/shipit/2_prstatus.json'
with get_issue(datafile, statusfile) as iw:
iw.pr_files = [MockFile(u'lib/ansible/module_utils/foo/bar.py')]
# need to give the wrapper a list of known files to compare against
iw.gitrepo = GitRepoWrapperMock()
iw.gitrepo.files.append(u'lib/ansible/modules/foo/bar.py')
# predefine what the matcher is going to return
CM = ComponentMatcherMock()
CM.expected_results = [
{
u'repo_filename': u'lib/ansible/module_utils/foo/bar.py',
u'labels': [],
u'support': None,
u'maintainers': [u'ElsA', u'Oliver'],
u'notify': [u'ElsA', u'Oliver'],
u'ignore': [],
}
]
meta = self.meta.copy()
iw._commits = []
meta.update(get_component_match_facts(iw, CM, []))
facts = get_shipit_facts(iw, meta, botmeta_files, core_team=[u'bcoca', u'mscherer'], botnames=[u'ansibot'])
self.assertEqual(iw.submitter, u'ElsA')
self.assertTrue(facts[u'owner_pr'])
def test_owner_pr_module_utils_and_modules_updated_submitter_maintainer_2(self):
"""
PR updates 2 files (one below modules, the other below module_utils),
submitter is a maintainer from both, check that owner_pr is set.
Submitter is maintainer from module_utils file.
"""
BOTMETA = u"""
---
macros:
modules: lib/ansible/modules
module_utils: lib/ansible/module_utils
files:
$modules/foo/bar.py:
maintainers: ElsA ZaZa
$module_utils/baz/bar.py:
maintainers: TiTi mscherer
"""
modules = {u'lib/ansible/modules/foo/bar.py': None}
module_indexer = create_indexer(textwrap.dedent(BOTMETA), modules)
self.assertEqual(len(module_indexer.modules), 2) # ensure only fake data are loaded
self.assertEqual(sorted(module_indexer.botmeta[u'files'][u'lib/ansible/modules/foo/bar.py'][u'maintainers']), [u'ElsA', u'ZaZa'])
self.assertEqual(sorted(module_indexer.botmeta[u'files'][u'lib/ansible/module_utils/baz/bar.py'][u'maintainers']), [u'TiTi', u'mscherer'])
CM = ComponentMatcherMock()
CM.expected_results = [
{
u'repo_filename': u'lib/ansible/module_utils/foo/bar.py',
u'labels': [],
u'support': None,
u'maintainers': [u'ElsA', u'mscherer'],
u'notify': [u'ElsA', u'mscherer'],
u'ignore': [],
},
{
u'repo_filename': u'lib/ansible/modules/baz/bar.py',
u'labels': [],
u'support': None,
u'maintainers': [u'TiTi', u'ZaZa'],
u'notify': [u'TiTi', u'ZaZa'],
u'ignore': [],
}
]
meta = self.meta.copy()
datafile = u'tests/fixtures/shipit/0_issue.yml'
statusfile = u'tests/fixtures/shipit/0_prstatus.json'
with get_issue(datafile, statusfile) as iw:
iw.pr_files = [
MockFile(u'lib/ansible/modules/foo/bar.py'),
MockFile(u'lib/ansible/module_utils/baz/bar.py')
]
iw.file_indexer = FileIndexerMock()
meta.update(get_component_match_facts(iw, CM, []))
facts = get_shipit_facts(iw, meta, module_indexer, core_team=[u'bcoca'], botnames=[u'ansibot'])
self.assertEqual(iw.submitter, u'mscherer')
self.assertFalse(facts[u'owner_pr'])
def test_owner_pr_module_utils_and_modules_updated_submitter_maintainer_1(
self):
"""
PR updates 2 files (one below modules, the other below module_utils),
submitter is a maintainer from both, check that owner_pr is set.
Submitter is maintainer from module file.
"""
botmeta_files = {
'lib/ansible/modules/foo/bar.py': {
'maintainers': ['ElsA', 'mscherer']
},
'lib/ansible/module_utils/baz/bar.py': {
'maintainers': ['TiTi', 'ZaZa']
},
}
CM = ComponentMatcherMock()
CM.expected_results = [{
'repo_filename': 'lib/ansible/module_utils/foo/bar.py',
'labels': [],
'support': None,
'maintainers': ['ElsA', 'mscherer'],
'notify': ['ElsA', 'mscherer'],
'ignore': [],
}, {
'repo_filename': 'lib/ansible/modules/baz/bar.py',
'labels': [],
'support': None,
'maintainers': ['TiTi', 'ZaZa'],
'notify': ['TiTi', 'ZaZa'],
'ignore': [],
}]
meta = self.meta.copy()
datafile = 'tests/fixtures/shipit/0_issue.yml'
statusfile = 'tests/fixtures/shipit/0_prstatus.json'
with get_issue(datafile, statusfile) as iw:
iw.pr_files = [
MockFile('lib/ansible/modules/foo/bar.py'),
MockFile('lib/ansible/module_utils/baz/bar.py')
]
iw.gitrepo = GitRepoWrapperMock()
iw.gitrepo.files = [
'lib/ansible/modules/foo/bar.py',
'lib/ansible/module_utils/baz/bar.py'
]
iw._commits = []
meta.update(get_component_match_facts(iw, CM, []))
facts = get_shipit_facts(iw,
meta,
botmeta_files,
core_team=['bcoca', 'mscherer'],
botnames=['ansibot'])
self.assertEqual(iw.submitter, 'mscherer')
self.assertFalse(facts['owner_pr'])
def test_owner_pr_submitter_is_maintainer_one_module_file_updated_changelog(
self):
"""
Submitter is a maintainer: ensure owner_pr is set even if changelog fragment is present
"""
BOTMETA = u"""
---
macros:
modules: lib/ansible/modules
files:
$modules/foo/bar.py:
maintainers: ElsA Oliver
"""
modules = {u'lib/ansible/modules/foo/bar.py': None}
module_indexer = create_indexer(textwrap.dedent(BOTMETA), modules)
self.assertEqual(len(module_indexer.modules),
2) # ensure only fake data are loaded
self.assertEqual(
sorted(module_indexer.botmeta[u'files']
[u'lib/ansible/modules/foo/bar.py'][u'maintainers']),
[u'ElsA', u'Oliver'])
datafile = u'tests/fixtures/shipit/2_issue.yml'
statusfile = u'tests/fixtures/shipit/2_prstatus.json'
with get_issue(datafile, statusfile) as iw:
iw.pr_files = [
MockFile(u'lib/ansible/modules/foo/bar.py'),
MockFile(u'changelogs/fragments/00000-fragment.yaml')
]
# need to give the wrapper a list of known files to compare against
iw.file_indexer = FileIndexerMock()
iw.file_indexer.files.append(u'lib/ansible/modules/foo/bar.py')
# predefine what the matcher is going to return
CM = ComponentMatcherMock()
CM.expected_results = [{
u'repo_filename': u'lib/ansible/modules/foo/bar.py',
u'labels': [],
u'support': None,
u'maintainers': [u'ElsA', u'Oliver'],
u'notify': [u'ElsA', u'Oliver'],
u'ignore': [],
}]
meta = self.meta.copy()
iw._commits = []
meta.update(get_component_match_facts(iw, CM, []))
facts = get_shipit_facts(iw,
meta,
module_indexer,
core_team=[u'bcoca', u'mscherer'],
botnames=[u'ansibot'])
self.assertEqual(iw.submitter, u'ElsA')
self.assertTrue(facts[u'owner_pr'])
def test_owner_pr_submitter_is_maintainer_one_module_utils_file_updated(
self):
"""
Submitter is a maintainer: ensure owner_pr is set (only one file below module_utils updated)
"""
BOTMETA = """
---
macros:
modules: lib/ansible/modules
module_utils: lib/ansible/module_utils
files:
$module_utils/foo/bar.py:
maintainers: ElsA Oliver
"""
modules = {'lib/ansible/module_utils/foo/bar.py': None}
module_indexer = create_indexer(textwrap.dedent(BOTMETA), modules)
self.assertEqual(len(module_indexer.modules),
2) # ensure only fake data are loaded
self.assertEqual(
sorted(module_indexer.botmeta['files']
['lib/ansible/module_utils/foo/bar.py']['maintainers']),
['ElsA', 'Oliver'])
issue = IssueMock('/dev/null')
issue.user.login = '******'
issue.html_url = 'https://github.com/ansible/ansible/pull/123'
iw = IssueWrapper(cachedir="", issue=issue)
iw.pr_files = [MockFile('lib/ansible/module_utils/foo/bar.py')]
# need to give the wrapper a list of known files to compare against
iw.file_indexer = FileIndexerMock()
# predefine what the matcher is going to return
CM = ComponentMatcherMock()
CM.expected_results = [{
'repo_filename': 'lib/ansible/module_utils/foo/bar.py',
'labels': [],
'support': None,
'maintainers': ['ElsA', 'Oliver'],
'notify': ['ElsA', 'Oliver'],
'ignore': [],
}]
meta = self.meta.copy()
meta.update(
get_component_match_facts(iw, meta, CM, FileIndexerMock(),
module_indexer, []))
facts = get_shipit_facts(iw,
meta,
module_indexer,
core_team=['bcoca', 'mscherer'],
botnames=['ansibot'])
self.assertEqual(iw.submitter, 'ElsA')
self.assertTrue(facts['owner_pr'])
def test_owner_pr_submitter_is_maintainer_new_module(self):
"""
Submitter is a maintainer: pull request adds a new module: ensure owner_pr is False
"""
BOTMETA = """
---
macros:
modules: lib/ansible/modules
module_utils: lib/ansible/module_utils
files:
$modules/foo/bar.py:
maintainers: ElsA mscherer
"""
modules = {} # new module
module_indexer = create_indexer(textwrap.dedent(BOTMETA), modules)
self.assertEqual(len(module_indexer.modules),
1) # ensure only fake data are loaded
# Ensure that BOTMETA.yml updates doesn't interfere
self.assertEqual(
sorted(module_indexer.botmeta['files']
['lib/ansible/modules/foo/bar.py']['maintainers']),
['ElsA', 'mscherer'])
CM = ComponentMatcherMock()
CM.expected_results = [{
'repo_filename': 'lib/ansible/modules/foo/bar.py',
'labels': [],
'support': None,
'maintainers': ['ElsA', 'mscherer'],
'notify': ['ElsA', 'mscherer'],
'ignore': [],
}]
meta = self.meta.copy()
datafile = 'tests/fixtures/shipit/0_issue.yml'
statusfile = 'tests/fixtures/shipit/0_prstatus.json'
with get_issue(datafile, statusfile) as iw:
iw.pr_files = [MockFile('lib/ansible/modules/foo/bar.py')]
iw.file_indexer = FileIndexerMock()
#iw.file_indexer.files.append('lib/ansible/modules/foo/bar.py')
meta.update(
get_component_match_facts(iw, {}, CM, iw.file_indexer,
module_indexer, []))
facts = get_shipit_facts(iw,
meta,
module_indexer,
core_team=['bcoca'],
botnames=['ansibot'])
self.assertEqual(iw.submitter, 'mscherer')
self.assertFalse(facts['owner_pr'])
def test_owner_pr_submitter_is_not_maintainer_of_all_updated_files(self):
"""
PR updates 2 files below module_utils, submitter is a maintainer from only one: ensure owner_pr isn't set
"""
botmeta_files = {
'lib/ansible/module_utils/foo/bar.py': {
'maintainers': ['ElsA', 'Oliver']
},
'lib/ansible/module_utils/baz/bar.py': {
'maintainers': ['TiTi', 'ZaZa']
},
}
CM = ComponentMatcherMock()
CM.expected_results = [{
'repo_filename': 'lib/ansible/module_utils/foo/bar.py',
'labels': [],
'support': None,
'maintainers': ['ElsA', 'Oliver'],
'notify': ['ElsA', 'Oliver'],
'ignore': [],
}, {
'repo_filename': 'lib/ansible/modules/baz/bar.py',
'labels': [],
'support': None,
'maintainers': ['TiTi', 'ZaZa'],
'notify': ['TiTi', 'ZaZa'],
'ignore': [],
}]
issue = IssueMock('/dev/null')
issue.user.login = '******'
issue.html_url = 'https://github.com/ansible/ansible/pull/123'
cachedir = tempfile.mkdtemp()
gh = GithubWrapperMock()
iw = IssueWrapper(cachedir=cachedir, issue=issue, github=gh)
iw.pr_files = [
MockFile('lib/ansible/module_utils/foo/bar.py'),
MockFile('lib/ansible/module_utils/baz/bar.py')
]
iw.gitrepo = GitRepoWrapperMock()
iw.repo = MockRepo(repo_path='ansible/ansible')
meta = self.meta.copy()
iw._commits = []
meta.update(get_component_match_facts(iw, CM, []))
facts = get_shipit_facts(iw,
meta,
botmeta_files,
core_team=['bcoca', 'mscherer'],
botnames=['ansibot'])
shutil.rmtree(cachedir)
self.assertEqual(iw.submitter, 'ElsA')
self.assertFalse(facts['owner_pr'])
def test_owner_pr_module_utils_and_modules_updated_submitter_maintainer_2(
self):
"""
PR updates 2 files (one below modules, the other below module_utils),
submitter is a maintainer from both, check that owner_pr is set.
Submitter is maintainer from module_utils file.
"""
BOTMETA = """
---
macros:
modules: lib/ansible/modules
module_utils: lib/ansible/module_utils
files:
$modules/foo/bar.py:
maintainers: ElsA ZaZa
$module_utils/baz/bar.py:
maintainers: TiTi mscherer
"""
modules = {'lib/ansible/modules/foo/bar.py': None}
module_indexer = create_indexer(textwrap.dedent(BOTMETA), modules)
self.assertEqual(len(module_indexer.modules),
2) # ensure only fake data are loaded
self.assertEqual(
sorted(module_indexer.botmeta['files']
['lib/ansible/modules/foo/bar.py']['maintainers']),
['ElsA', 'ZaZa'])
self.assertEqual(
sorted(module_indexer.botmeta['files']
['lib/ansible/module_utils/baz/bar.py']['maintainers']),
['TiTi', 'mscherer'])
meta = self.meta.copy()
datafile = 'tests/fixtures/shipit/0_issue.yml'
statusfile = 'tests/fixtures/shipit/0_prstatus.json'
with get_issue(datafile, statusfile) as iw:
iw.pr_files = [
MockFile('lib/ansible/modules/foo/bar.py'),
MockFile('lib/ansible/module_utils/baz/bar.py')
]
meta.update(
get_component_match_facts(iw, {}, FileIndexerMock(),
module_indexer, []))
facts = get_shipit_facts(iw,
meta,
module_indexer,
core_team=['bcoca'],
botnames=['ansibot'])
self.assertEqual(iw.submitter, 'mscherer')
self.assertFalse(facts['owner_pr'])
def test_review_facts_are_defined_module_utils(self):
BOTMETA = u"""
---
macros:
modules: lib/ansible/modules
module_utils: lib/ansible/module_utils
files:
$module_utils:
support: community
$modules/foo/bar.py:
maintainers: ElsA ZaZa
$module_utils/baz/bar.py:
maintainers: TiTi mscherer
"""
modules = {u'lib/ansible/modules/foo/bar.py': None}
module_indexer = create_indexer(textwrap.dedent(BOTMETA), modules)
self.assertEqual(len(module_indexer.modules), 2) # ensure only fake data are loaded
self.assertEqual(sorted(module_indexer.botmeta[u'files'][u'lib/ansible/modules/foo/bar.py'][u'maintainers']),[u'ElsA', u'ZaZa'])
self.assertEqual(sorted(module_indexer.botmeta[u'files'][u'lib/ansible/module_utils/baz/bar.py'][u'maintainers']),[u'TiTi', u'mscherer'])
datafile = u'tests/fixtures/shipit/2_issue.yml'
statusfile = u'tests/fixtures/shipit/2_prstatus.json'
with get_issue(datafile, statusfile) as iw:
iw.pr_files = [MockFile(u'lib/ansible/module_utils/foo/bar.py')]
# need to give the wrapper a list of known files to compare against
iw.file_indexer = FileIndexerMock()
iw.file_indexer.files.append(u'lib/ansible/modules/foo/bar.py')
# predefine what the matcher is going to return
CM = ComponentMatcherMock()
CM.expected_results = [
{
u'repo_filename': u'lib/ansible/module_utils/foo/bar.py',
u'labels': [],
u'support': None,
u'maintainers': [u'ElsA', u'Oliver'],
u'notify': [u'ElsA', u'Oliver'],
u'ignore': [],
}
]
meta = self.meta.copy()
iw._commits = []
meta.update(get_component_match_facts(iw, CM, []))
meta.update(get_shipit_facts(iw, meta, module_indexer, core_team=[u'bcoca'], botnames=[u'ansibot']))
facts = get_review_facts(iw, meta)
self.assertTrue(facts[u'community_review'])
self.assertFalse(facts[u'core_review'])
self.assertFalse(facts[u'committer_review'])
def run(self):
'''Emit an html report of each file and it's metadata'''
component_facts = {}
lblacklist = [
'ansible',
]
lblacklist = []
filenames = sorted(self.gitrepo.files)
filenames = [x for x in filenames if not x.startswith('/')]
filenames = [x for x in filenames if not '__pycache__' in x]
filenames = [x for x in filenames if not x.endswith('.pyo')]
filenames = [x for x in filenames if not x.endswith('.pyo')]
for fn in filenames:
logging.debug(fn)
meta = self.component_matcher.search_by_filepath(fn)
iw = IssueMock()
iw._component = fn
try:
facts = get_component_match_facts(iw, self.component_matcher,
[])
component_facts[fn] = {}
component_facts[fn]['component'] = facts['component_name']
component_facts[fn]['support'] = facts['component_matches'][0][
'support']
labels = facts['component_matches'][0]['labels'][:]
labels += facts['component_labels']
labels = sorted(set(labels))
labels = [x for x in labels if x not in lblacklist]
component_facts[fn]['labels'] = ','.join(labels)
component_facts[fn]['maintainers'] = \
','.join(facts['component_matches'][0]['maintainers'])
except Exception as e:
component_facts[fn] = {'error': '%s' % e}
logging.error(fn)
logging.error(e)
logging.info('done matching')
logging.info('building report in %s' % self.args.dest)
dest = os.path.expanduser(self.args.dest)
if os.path.exists(dest):
shutil.rmtree(dest)
skel = os.path.join(TEMPLATES, 'metareport')
shutil.copytree(skel, dest)
with open(os.path.join(dest, 'data.json'), 'w') as f:
f.write(json.dumps(list(component_facts.values())))
def test_owner_pr_submitter_is_not_maintainer_of_all_updated_files(self):
"""
PR updates 2 files below module_utils, submitter is a maintainer from only one: ensure owner_pr isn't set
"""
BOTMETA = """
---
macros:
modules: lib/ansible/modules
module_utils: lib/ansible/module_utils
files:
$module_utils/foo/bar.py:
maintainers: ElsA Oliver
$module_utils/baz/bar.py:
maintainers: TiTi ZaZa
"""
module_indexer = create_indexer(textwrap.dedent(BOTMETA), {})
self.assertEqual(len(module_indexer.modules),
1) # ensure only fake data are loaded
self.assertEqual(
sorted(module_indexer.botmeta['files']
['lib/ansible/module_utils/foo/bar.py']['maintainers']),
['ElsA', 'Oliver'])
self.assertEqual(
sorted(module_indexer.botmeta['files']
['lib/ansible/module_utils/baz/bar.py']['maintainers']),
['TiTi', 'ZaZa'])
issue = IssueMock('/dev/null')
issue.user.login = '******'
issue.html_url = 'https://github.com/ansible/ansible/pull/123'
iw = IssueWrapper(cachedir="", issue=issue)
iw.pr_files = [
MockFile('lib/ansible/module_utils/foo/bar.py'),
MockFile('lib/ansible/module_utils/baz/bar.py')
]
meta = self.meta.copy()
meta.update(
get_component_match_facts(iw, {}, FileIndexerMock(),
module_indexer, []))
facts = get_shipit_facts(iw,
meta,
module_indexer,
core_team=['bcoca', 'mscherer'],
botnames=['ansibot'])
self.assertEqual(iw.submitter, 'ElsA')
self.assertFalse(facts['owner_pr'])
def test_owner_pr_submitter_is_maintainer_new_module(self):
"""
Submitter is a maintainer: pull request adds a new module: ensure owner_pr is False
"""
BOTMETA = u"""
---
macros:
modules: lib/ansible/modules
module_utils: lib/ansible/module_utils
files:
$modules/foo/bar.py:
maintainers: ElsA mscherer
"""
modules = {} # new module
module_indexer = create_indexer(textwrap.dedent(BOTMETA), modules)
self.assertEqual(len(module_indexer.modules), 1) # ensure only fake data are loaded
# Ensure that BOTMETA.yml updates doesn't interfere
self.assertEqual(sorted(module_indexer.botmeta[u'files'][u'lib/ansible/modules/foo/bar.py'][u'maintainers']), [u'ElsA', u'mscherer'])
CM = ComponentMatcherMock()
CM.expected_results = [
{
u'repo_filename': u'lib/ansible/modules/foo/bar.py',
u'labels': [],
u'support': None,
u'maintainers': [u'ElsA', u'mscherer'],
u'notify': [u'ElsA', u'mscherer'],
u'ignore': [],
}
]
meta = self.meta.copy()
datafile = u'tests/fixtures/shipit/0_issue.yml'
statusfile = u'tests/fixtures/shipit/0_prstatus.json'
with get_issue(datafile, statusfile) as iw:
iw.pr_files = [MockFile(u'lib/ansible/modules/foo/bar.py')]
iw.file_indexer = FileIndexerMock()
meta.update(get_component_match_facts(iw, CM, []))
facts = get_shipit_facts(iw, meta, module_indexer, core_team=[u'bcoca'], botnames=[u'ansibot'])
self.assertEqual(iw.submitter, u'mscherer')
self.assertFalse(facts[u'owner_pr'])
def test_review_facts_are_defined_module_utils(self):
botmeta_files = {
'lib/ansible/module_utils': {
'support': 'community'
},
'lib/ansible/modules/foo/bar.py': {
'maintainers': ['ElsA', 'ZaZa']
},
'lib/ansible/module_utils/baz/bar.py': {
'maintainers': ['TiTi', 'mscherer']
},
}
datafile = 'tests/fixtures/shipit/2_issue.yml'
statusfile = 'tests/fixtures/shipit/2_prstatus.json'
with get_issue(datafile, statusfile) as iw:
iw.pr_files = [MockFile('lib/ansible/module_utils/foo/bar.py')]
# need to give the wrapper a list of known files to compare against
iw.gitrepo = GitRepoWrapperMock()
iw.gitrepo.files.append('lib/ansible/modules/foo/bar.py')
# predefine what the matcher is going to return
CM = ComponentMatcherMock()
CM.expected_results = [{
'repo_filename': 'lib/ansible/module_utils/foo/bar.py',
'labels': [],
'support': None,
'maintainers': ['ElsA', 'Oliver'],
'notify': ['ElsA', 'Oliver'],
'ignore': [],
}]
meta = self.meta.copy()
iw._commits = []
meta.update(get_component_match_facts(iw, CM, []))
meta.update(
get_shipit_facts(iw,
meta,
botmeta_files,
core_team=['bcoca'],
botnames=['ansibot']))
facts = get_review_facts(iw, meta)
self.assertTrue(facts['community_review'])
self.assertFalse(facts['core_review'])
self.assertFalse(facts['committer_review'])
def test_owner_pr_submitter_is_maintainer_one_module_utils_file_updated(
self):
"""
Submitter is a maintainer: ensure owner_pr is set (only one file below module_utils updated)
"""
BOTMETA = """
---
macros:
modules: lib/ansible/modules
module_utils: lib/ansible/module_utils
files:
$module_utils/foo/bar.py:
maintainers: ElsA Oliver
"""
modules = {'lib/ansible/module_utils/foo/bar.py': None}
module_indexer = create_indexer(textwrap.dedent(BOTMETA), modules)
self.assertEqual(len(module_indexer.modules),
2) # ensure only fake data are loaded
self.assertEqual(
sorted(module_indexer.botmeta['files']
['lib/ansible/module_utils/foo/bar.py']['maintainers']),
['ElsA', 'Oliver'])
issue = IssueMock('/dev/null')
issue.user.login = '******'
issue.html_url = 'https://github.com/ansible/ansible/pull/123'
iw = IssueWrapper(cachedir="", issue=issue)
iw.pr_files = [MockFile('lib/ansible/module_utils/foo/bar.py')]
meta = self.meta.copy()
meta.update(
get_component_match_facts(iw, None, FileIndexerMock(),
module_indexer, []))
facts = get_shipit_facts(iw,
meta,
module_indexer,
core_team=['bcoca', 'mscherer'],
botnames=['ansibot'])
self.assertEqual(iw.submitter, 'ElsA')
self.assertTrue(facts['owner_pr'])
def test_owner_pr_submitter_is_maintainer_one_module_file_updated_changelog(
self):
"""
Submitter is a maintainer: ensure owner_pr is set even if changelog fragment is present
"""
botmeta_files = {
'lib/ansible/modules/foo/bar.py': {
'maintainers': ['ElsA', 'Oliver']
}
}
datafile = 'tests/fixtures/shipit/2_issue.yml'
statusfile = 'tests/fixtures/shipit/2_prstatus.json'
with get_issue(datafile, statusfile) as iw:
iw._pr_files = [
MockFile('lib/ansible/modules/foo/bar.py'),
MockFile('changelogs/fragments/00000-fragment.yaml')
]
# need to give the wrapper a list of known files to compare against
iw.gitrepo = GitRepoWrapperMock()
iw.gitrepo.files.append('lib/ansible/modules/foo/bar.py')
# predefine what the matcher is going to return
CM = ComponentMatcherMock()
CM.expected_results = [{
'repo_filename': 'lib/ansible/modules/foo/bar.py',
'labels': [],
'support': None,
'maintainers': ['ElsA', 'Oliver'],
'notify': ['ElsA', 'Oliver'],
'ignore': [],
}]
meta = self.meta.copy()
iw._commits = []
meta.update(get_component_match_facts(iw, CM, []))
facts = get_shipit_facts(iw,
meta,
botmeta_files,
core_team=['bcoca', 'mscherer'],
botnames=['ansibot'])
self.assertEqual(iw.submitter, 'ElsA')
self.assertTrue(facts['owner_pr'])
def test_owner_pr_submitter_is_maintainer_one_modules_file_updated(self):
"""
Submitter is a maintainer: ensure owner_pr is set (only one file below modules updated)
"""
BOTMETA = """
---
macros:
modules: lib/ansible/modules
module_utils: lib/ansible/module_utils
files:
$modules/foo/bar.py:
maintainers: ElsA mscherer
"""
modules = {'lib/ansible/modules/foo/bar.py': None}
module_indexer = create_indexer(textwrap.dedent(BOTMETA), modules)
self.assertEqual(len(module_indexer.modules),
2) # ensure only fake data are loaded
self.assertEqual(
sorted(module_indexer.botmeta['files']
['lib/ansible/modules/foo/bar.py']['maintainers']),
['ElsA', 'mscherer'])
meta = self.meta.copy()
datafile = 'tests/fixtures/shipit/0_issue.yml'
statusfile = 'tests/fixtures/shipit/0_prstatus.json'
with get_issue(datafile, statusfile) as iw:
iw.pr_files = [MockFile('lib/ansible/modules/foo/bar.py')]
meta.update(
get_component_match_facts(iw, {}, FileIndexerMock(),
module_indexer, []))
facts = get_shipit_facts(iw,
meta,
module_indexer,
core_team=['bcoca'],
botnames=['ansibot'])
self.assertEqual(iw.submitter, 'mscherer')
self.assertTrue(facts['owner_pr'])
def test_owner_pr_submitter_is_maintainer_one_modules_file_updated(self):
"""
Submitter is a maintainer: ensure owner_pr is set (only one file below modules updated)
"""
botmeta_files = {
'lib/ansible/modules/foo/bar.py': {
'maintainers': ['ElsA', 'mscherer']
}
}
CM = ComponentMatcherMock()
CM.expected_results = [{
'repo_filename': 'lib/ansible/modules/foo/bar.py',
'labels': [],
'support': None,
'maintainers': ['ElsA', 'mscherer'],
'notify': ['ElsA', 'mscherer'],
'ignore': [],
}]
meta = self.meta.copy()
datafile = 'tests/fixtures/shipit/0_issue.yml'
statusfile = 'tests/fixtures/shipit/0_prstatus.json'
with get_issue(datafile, statusfile) as iw:
iw.pr_files = [MockFile('lib/ansible/modules/foo/bar.py')]
iw.gitrepo = GitRepoWrapperMock()
iw.gitrepo.files.append('lib/ansible/modules/foo/bar.py')
meta.update(get_component_match_facts(iw, CM, []))
facts = get_shipit_facts(iw,
meta,
botmeta_files,
core_team=['bcoca'],
botnames=['ansibot'])
self.assertEqual(iw.submitter, 'mscherer')
self.assertTrue(facts['owner_pr'])
def main():
set_logger()
METAFILES = extract_metafiles()
SKIP = load_skip()
EXPECTED = load_expected()
MATCH_MAP = load_match_map()
ERRORS = []
ERRORS_COMPONENTS = []
start_at = None
if len(sys.argv) == 2:
start_at = int(sys.argv[1])
FI = FileIndexer(checkoutdir=CACHEDIR)
with open('/tmp/files.json', 'wb') as f:
f.write(json.dumps(FI.files, indent=2))
GQLC = GithubGraphQLClient(C.DEFAULT_GITHUB_TOKEN)
MI = ModuleIndexer(cachedir=CACHEDIR,
gh_client=GQLC,
blames=False,
commits=False)
CM = AnsibleComponentMatcher(cachedir=CACHEDIR)
for k, v in MI.modules.items():
if k in MATCH_MAP:
MATCH_MAP.pop(k, None)
kname = v.get('name')
if kname not in MATCH_MAP:
MATCH_MAP[kname] = v.get('repo_filename')
if kname + ' module' not in MATCH_MAP:
MATCH_MAP[kname + ' module'] = v.get('repo_filename')
if kname + 'module: ' + kname not in MATCH_MAP:
MATCH_MAP['module: ' + kname] = v.get('repo_filename')
if kname + 'module ' + kname not in MATCH_MAP:
MATCH_MAP['module ' + kname] = v.get('repo_filename')
# /modules/remote_management/foreman/katello.py
pname = k.replace('lib/ansible', '')
if pname not in MATCH_MAP:
MATCH_MAP[pname] = v.get('repo_filename')
# ansible/modules/packaging/os/rpm_key.py
pname = k.replace('lib/', '/')
if pname not in MATCH_MAP:
MATCH_MAP[pname] = v.get('repo_filename')
# /ansible/modules/packaging/os/rpm_key.py
pname = k.replace('lib/', '')
if pname not in MATCH_MAP:
MATCH_MAP[pname] = v.get('repo_filename')
# ansible/lib/ansible/modules/monitoring/monit.py
pname = 'ansible/' + k
if pname not in MATCH_MAP:
MATCH_MAP[pname] = v.get('repo_filename')
# network/f5/bigip_gtm_wide_ip
pname = k.replace('lib/ansible/modules/', '')
pname = pname.replace('.py', '')
pname = pname.replace('.ps1', '')
if pname not in MATCH_MAP:
MATCH_MAP[pname] = v.get('repo_filename')
# network/f5/bigip_gtm_wide_ip.py
pname = k.replace('lib/ansible/modules/', '')
if pname not in MATCH_MAP:
MATCH_MAP[pname] = v.get('repo_filename')
# modules/packaging/os/pkgng.py
pname = k.replace('lib/ansible/', '')
if pname not in MATCH_MAP:
MATCH_MAP[pname] = v.get('repo_filename')
save_match_map(MATCH_MAP)
total = len(METAFILES)
for IDMF, MF in enumerate(METAFILES):
if start_at and IDMF < start_at:
continue
with open(MF, 'rb') as f:
meta = json.loads(f.read())
if not meta.get('is_issue'):
continue
component = meta.get('template_data', {}).get('component_raw')
#if component != 'Module `synchronize`':
#if component != 'Module: include_role':
# continue
if component:
print(f'------------------------------------------ {total}|{IDMF}')
print(meta['html_url'])
print(meta['title'])
print(component)
hurl = meta['html_url']
if hurl in SKIP:
continue
# bad template or bad template parsing
if len(component) > 100:
continue
iw = IssueWrapperMock(meta)
if 'module' not in iw.body.lower(
) and 'module' not in iw.title.lower():
continue
expected_fns = []
# OLD METHOD
if hurl not in EXPECTED and component not in MATCH_MAP:
cmf = get_component_match_facts(iw, meta, FI, MI, LABELS)
expected_fns = cmf.get('module_match')
if not isinstance(expected_fns, list):
expected_fns = [expected_fns]
expected_fns = [x['repo_filename'] for x in expected_fns if x]
if 'component_matches' in cmf:
expected_fns = [
x['filename'] for x in cmf['component_matches']
]
expected_fns = sorted(set(expected_fns))
# NEW METHOD
cmr = CM.match_components(iw.title, iw.body,
iw.template_data.get('component_raw'))
cmr_fns = [x['repo_filename'] for x in cmr if x]
cmr_fns = sorted(set(cmr_fns))
# VALIDATE FROM EXPECTED IF KNOWN
if hurl in EXPECTED:
if EXPECTED[hurl] and not isinstance(EXPECTED[hurl], list):
expected_fns = [EXPECTED[hurl]]
elif EXPECTED[hurl]:
expected_fns = EXPECTED[hurl]
else:
expected_fns = []
# USE THE CACHED MAP
if component in MATCH_MAP:
expected_fns = MATCH_MAP[component]
if not isinstance(expected_fns, list):
expected_fns = [expected_fns]
elif component.lower() in MATCH_MAP:
expected_fns = MATCH_MAP[component.lower()]
if not isinstance(expected_fns, list):
expected_fns = [expected_fns]
elif component.startswith(':\n') and component.endswith(' module'):
mapkey = component.lstrip(':\n')
if mapkey in MATCH_MAP:
expected_fns = MATCH_MAP[mapkey]
if not isinstance(expected_fns, list):
expected_fns = [expected_fns]
# OLD CODE USED ACTION PLUGINS INSTEAD OF MODULES
if expected_fns != cmr_fns and hurl not in EXPECTED:
if len(expected_fns) == 1 and len(
cmr_fns) == 1 and 'plugins/action' in expected_fns[0]:
e_bn = os.path.basename(expected_fns[0])
c_bn = os.path.basename(cmr_fns[0])
if e_bn == c_bn:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
# DOCS URLS
if expected_fns != cmr_fns and hurl not in EXPECTED:
if len(cmr_fns) == 1 and 'lib/ansible/modules' in cmr_fns[0]:
c_bn = os.path.basename(cmr_fns[0])
if f'docs.ansible.com/ansible/latest/{c_bn}_module.html' in component:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
elif CM.strategy in ['search_by_regex_urls']:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
# NXOS ISSUES HAVE NXOS_VERSION HEADER
if '- nxos' in component:
if len(cmr_fns) == 1:
if os.path.basename(cmr_fns[0]).replace('.py',
'') in component:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
#import epdb; epdb.st()
# ODDBALL MODULE COMPONENTS
if len(cmr_fns) == 1 and 'lib/ansible/modules' in cmr_fns[0]:
bn = os.path.basename(cmr_fns[0])
bn = bn.replace('.py', '')
bn = bn.replace('.ps1', '')
if (bn in component or bn.lstrip('_')
in component) and 'module' in component.lower():
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
elif component == '- ' + bn:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
elif component == bn + '.py' or component == bn + '.ps1':
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
elif component == '_' + bn + '.py' or component == '_' + bn + '.ps1':
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
elif component == ':\n' + bn or component == ':\n' + bn.lstrip(
'_'):
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
# 'multiple modules', etc ...
if component in CM.KEYWORDS or component.lower() in CM.KEYWORDS:
if component in CM.KEYWORDS and CM.KEYWORDS[
component] is None and not cmr_fns:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
elif component.lower() in CM.KEYWORDS and CM.KEYWORDS[
component.lower()] is None and not cmr_fns:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
elif len(cmr_fns) == 1 and cmr_fns[0] == CM.KEYWORDS.get(
component):
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
elif len(cmr_fns) == 1 and cmr_fns[0] == CM.KEYWORDS.get(
component.lower()):
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
if component.lstrip('-').strip() in CM.KEYWORDS and len(
cmr_fns) == 1:
cname = component.lstrip('-').strip()
if CM.KEYWORDS[cname] == cmr_fns[0]:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
if component.endswith(' lookup') and len(
cmr_fns
) == 1 and 'lib/ansible/plugins/lookup' in cmr_fns[0]:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
if component.endswith(' inventory script') and len(
cmr_fns) == 1 and 'contrib/inventory' in cmr_fns[0]:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
if component.startswith('ansible/lib') and len(cmr_fns) == 1:
fn = cmr_fns[0]
if 'ansible/' + fn == component:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
if component.endswith(' inventory plugin') and len(cmr_fns) == 1:
fn = cmr_fns[0]
if fn.startswith('lib/ansible/plugins/inventory'):
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
if component == 'ec2.py' and cmr_fns and 'contrib/inventory/ec2.py' in cmr_fns:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
if len(expected_fns) == 1 and len(cmr_fns) == 1:
if os.path.basename(expected_fns[0]) == os.path.basename(
cmr_fns[0]):
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
# COMPARE AND RECORD
if expected_fns != cmr_fns and hurl not in EXPECTED:
if component in MATCH_MAP or component.lower() in MATCH_MAP:
if component.lower() in MATCH_MAP:
mmc = MATCH_MAP[component.lower()]
else:
mmc = MATCH_MAP[component]
if not isinstance(mmc, list):
mmc == [mmc]
if mmc == cmr_fns:
EXPECTED[iw.html_url] = cmr_fns
save_expected(EXPECTED)
continue
print('## COMPONENT ...')
print(component)
print('## EXPECTED ...')
pprint(expected_fns)
print('## RESULT ...')
pprint(cmr_fns)
print('## STRATEGIES ..')
pprint(CM.strategy)
pprint(CM.strategies)
print('--------------------------------')
res = raw_input('Is the result correct? (y/n/s/d): ')
if res.lower() in ['y', 'yes']:
MATCH_MAP[component] = cmr_fns
EXPECTED[iw.html_url] = cmr_fns
save_expected(EXPECTED)
continue
elif res.lower() in ['s', 'skip']:
SKIP.append(hurl)
save_skip(SKIP)
continue
elif res.lower() in ['d', 'debug']:
import epdb
epdb.st()
ERRORS.append(iw.html_url)
ERRORS_COMPONENTS.append({
'url':
iw.html_url,
'component':
component,
'component_raw':
iw.template_data.get('component_raw'),
'result':
cmr_fns,
'expected':
expected_fns,
'strategy':
CM.strategy,
'strategies':
CM.strategies
})
else:
if component not in MATCH_MAP:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
if hurl not in EXPECTED:
EXPECTED[hurl] = cmr_fns
save_expected(EXPECTED)
continue
pprint(ERRORS)
fn = os.path.join(FIXTUREDIR, 'component_errors.json')
with open(fn, 'wb') as f:
f.write(json.dumps(ERRORS_COMPONENTS, indent=2, sort_keys=True))
clean_metafiles(METAFILES)
def main():
set_logger()
METAFILES = extract_metafiles()
SKIP = load_skip()
EXPECTED = load_expected()
MATCH_MAP = load_match_map()
ERRORS = []
ERRORS_COMPONENTS = []
start_at = None
if len(sys.argv) == 2:
start_at = int(sys.argv[1])
FI = FileIndexer(checkoutdir=CACHEDIR)
with open('/tmp/files.json', 'wb') as f:
f.write(json.dumps(FI.files, indent=2))
GQLC = GithubGraphQLClient(C.DEFAULT_GITHUB_TOKEN)
#GWS = GithubWebScraper(cachedir=CACHEDIR)
MI = ModuleIndexer(cachedir=CACHEDIR, gh_client=GQLC, blames=False, commits=False)
#CM = ComponentMatcher(cachedir=CACHEDIR, module_indexer=MI, file_indexer=FI)
CM = AnsibleComponentMatcher(cachedir=CACHEDIR)
for k,v in MI.modules.items():
if k in MATCH_MAP:
MATCH_MAP.pop(k, None)
kname = v.get('name')
if kname not in MATCH_MAP:
MATCH_MAP[kname] = v.get('repo_filename')
if kname + ' module' not in MATCH_MAP:
MATCH_MAP[kname + ' module'] = v.get('repo_filename')
if kname + 'module: ' + kname not in MATCH_MAP:
MATCH_MAP['module: ' + kname] = v.get('repo_filename')
if kname + 'module ' + kname not in MATCH_MAP:
MATCH_MAP['module ' + kname] = v.get('repo_filename')
# /modules/remote_management/foreman/katello.py
pname = k.replace('lib/ansible', '')
if pname not in MATCH_MAP:
MATCH_MAP[pname] = v.get('repo_filename')
# ansible/modules/packaging/os/rpm_key.py
pname = k.replace('lib/', '/')
if pname not in MATCH_MAP:
MATCH_MAP[pname] = v.get('repo_filename')
# /ansible/modules/packaging/os/rpm_key.py
pname = k.replace('lib/', '')
if pname not in MATCH_MAP:
MATCH_MAP[pname] = v.get('repo_filename')
# ansible/lib/ansible/modules/monitoring/monit.py
pname = 'ansible/' + k
if pname not in MATCH_MAP:
MATCH_MAP[pname] = v.get('repo_filename')
# network/f5/bigip_gtm_wide_ip
pname = k.replace('lib/ansible/modules/', '')
pname = pname.replace('.py', '')
pname = pname.replace('.ps1', '')
if pname not in MATCH_MAP:
MATCH_MAP[pname] = v.get('repo_filename')
# network/f5/bigip_gtm_wide_ip.py
pname = k.replace('lib/ansible/modules/', '')
if pname not in MATCH_MAP:
MATCH_MAP[pname] = v.get('repo_filename')
# modules/packaging/os/pkgng.py
pname = k.replace('lib/ansible/', '')
if pname not in MATCH_MAP:
MATCH_MAP[pname] = v.get('repo_filename')
save_match_map(MATCH_MAP)
total = len(METAFILES)
for IDMF,MF in enumerate(METAFILES):
if start_at and IDMF < start_at:
continue
with open(MF, 'rb') as f:
meta = json.loads(f.read())
if not meta.get('is_issue'):
continue
component = meta.get('template_data', {}).get('component_raw')
#if component != 'Module `synchronize`':
#if component != 'Module: include_role':
# continue
if component:
print('------------------------------------------ {}|{}'.format(total, IDMF))
print(meta['html_url'])
print(meta['title'])
print(component)
hurl = meta['html_url']
if hurl in SKIP:
continue
# bad template or bad template parsing
if len(component) > 100:
continue
iw = IssueWrapperMock(meta)
if 'module' not in iw.body.lower() and 'module' not in iw.title.lower():
continue
expected_fns = []
# OLD METHOD
if hurl not in EXPECTED and component not in MATCH_MAP:
cmf = get_component_match_facts(iw, meta, FI, MI, LABELS)
expected_fns = cmf.get('module_match')
if not isinstance(expected_fns, list):
expected_fns = [expected_fns]
expected_fns = [x['repo_filename'] for x in expected_fns if x]
if 'component_matches' in cmf:
expected_fns = [x['filename'] for x in cmf['component_matches']]
expected_fns = sorted(set(expected_fns))
# NEW METHOD
cmr = CM.match_components(iw.title, iw.body, iw.template_data.get('component_raw'))
cmr_fns = [x['repo_filename'] for x in cmr if x]
cmr_fns = sorted(set(cmr_fns))
# VALIDATE FROM EXPECTED IF KNOWN
if hurl in EXPECTED:
if EXPECTED[hurl] and not isinstance(EXPECTED[hurl], list):
expected_fns = [EXPECTED[hurl]]
elif EXPECTED[hurl]:
expected_fns = EXPECTED[hurl]
else:
expected_fns = []
# USE THE CACHED MAP
if component in MATCH_MAP:
expected_fns = MATCH_MAP[component]
if not isinstance(expected_fns, list):
expected_fns = [expected_fns]
elif component.lower() in MATCH_MAP:
expected_fns = MATCH_MAP[component.lower()]
if not isinstance(expected_fns, list):
expected_fns = [expected_fns]
elif component.startswith(':\n') and component.endswith(' module'):
mapkey = component.lstrip(':\n')
if mapkey in MATCH_MAP:
expected_fns = MATCH_MAP[mapkey]
if not isinstance(expected_fns, list):
expected_fns = [expected_fns]
# OLD CODE USED ACTION PLUGINS INSTEAD OF MODULES
if expected_fns != cmr_fns and hurl not in EXPECTED:
if len(expected_fns) == 1 and len(cmr_fns) == 1 and 'plugins/action' in expected_fns[0]:
e_bn = os.path.basename(expected_fns[0])
c_bn = os.path.basename(cmr_fns[0])
if e_bn == c_bn:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
# DOCS URLS
if expected_fns != cmr_fns and hurl not in EXPECTED:
if len(cmr_fns) == 1 and 'lib/ansible/modules' in cmr_fns[0]:
c_bn = os.path.basename(cmr_fns[0])
if 'docs.ansible.com/ansible/latest/{}_module.html'.format(c_bn) in component:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
elif CM.strategy in ['search_by_regex_urls']:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
# NXOS ISSUES HAVE NXOS_VERSION HEADER
if '- nxos' in component:
if len(cmr_fns) == 1:
if os.path.basename(cmr_fns[0]).replace('.py', '') in component:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
#import epdb; epdb.st()
# ODDBALL MODULE COMPONENTS
if len(cmr_fns) == 1 and 'lib/ansible/modules' in cmr_fns[0]:
bn = os.path.basename(cmr_fns[0])
bn = bn.replace('.py', '')
bn = bn.replace('.ps1', '')
if (bn in component or bn.lstrip('_') in component) and 'module' in component.lower():
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
elif component == '- ' + bn:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
elif component == bn + '.py' or component == bn + '.ps1':
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
elif component == '_' + bn + '.py' or component == '_' + bn + '.ps1':
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
elif component == ':\n' + bn or component == ':\n' + bn.lstrip('_'):
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
# 'multiple modules', etc ...
if component in CM.KEYWORDS or component.lower() in CM.KEYWORDS:
if component in CM.KEYWORDS and CM.KEYWORDS[component] is None and not cmr_fns:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
elif component.lower() in CM.KEYWORDS and CM.KEYWORDS[component.lower()] is None and not cmr_fns:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
elif len(cmr_fns) == 1 and cmr_fns[0] == CM.KEYWORDS.get(component):
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
elif len(cmr_fns) == 1 and cmr_fns[0] == CM.KEYWORDS.get(component.lower()):
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
if component.lstrip('-').strip() in CM.KEYWORDS and len(cmr_fns) == 1:
cname = component.lstrip('-').strip()
if CM.KEYWORDS[cname] == cmr_fns[0]:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
if component.endswith(' lookup') and len(cmr_fns) == 1 and 'lib/ansible/plugins/lookup' in cmr_fns[0]:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
if component.endswith(' inventory script') and len(cmr_fns) == 1 and 'contrib/inventory' in cmr_fns[0]:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
if component.startswith('ansible/lib') and len(cmr_fns) == 1:
fn = cmr_fns[0]
if 'ansible/' + fn == component:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
if component.endswith(' inventory plugin') and len(cmr_fns) == 1:
fn = cmr_fns[0]
if fn.startswith('lib/ansible/plugins/inventory'):
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
if component == 'ec2.py' and cmr_fns and 'contrib/inventory/ec2.py' in cmr_fns:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
if len(expected_fns) == 1 and len(cmr_fns) == 1:
if os.path.basename(expected_fns[0]) == os.path.basename(cmr_fns[0]):
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
# COMPARE AND RECORD
if expected_fns != cmr_fns and hurl not in EXPECTED:
if component in MATCH_MAP or component.lower() in MATCH_MAP:
if component.lower() in MATCH_MAP:
mmc = MATCH_MAP[component.lower()]
else:
mmc = MATCH_MAP[component]
if not isinstance(mmc, list):
mmc == [mmc]
if mmc == cmr_fns:
EXPECTED[iw.html_url] = cmr_fns
save_expected(EXPECTED)
continue
print('## COMPONENT ...')
print(component)
print('## EXPECTED ...')
pprint(expected_fns)
print('## RESULT ...')
pprint(cmr_fns)
print('## STRATEGIES ..')
pprint(CM.strategy)
pprint(CM.strategies)
print('--------------------------------')
res = raw_input('Is the result correct? (y/n/s/d): ')
if res.lower() in ['y', 'yes']:
MATCH_MAP[component] = cmr_fns
EXPECTED[iw.html_url] = cmr_fns
save_expected(EXPECTED)
continue
elif res.lower() in ['s', 'skip']:
SKIP.append(hurl)
save_skip(SKIP)
continue
elif res.lower() in ['d', 'debug']:
import epdb; epdb.st()
ERRORS.append(iw.html_url)
ERRORS_COMPONENTS.append(
{
'url': iw.html_url,
'component': component,
'component_raw': iw.template_data.get('component_raw'),
'result': cmr_fns,
'expected': expected_fns,
'strategy': CM.strategy,
'strategies': CM.strategies
}
)
else:
if component not in MATCH_MAP:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
if hurl not in EXPECTED:
EXPECTED[hurl] = cmr_fns
save_expected(EXPECTED)
continue
pprint(ERRORS)
fn = os.path.join(FIXTUREDIR, 'component_errors.json')
with open(fn, 'wb') as f:
f.write(json.dumps(ERRORS_COMPONENTS, indent=2, sort_keys=True))
clean_metafiles(METAFILES)
def main():
tocheck = [
#32226,
#30361,
#31006,
#58674,
#63611,
#64320,
#66891,
#68784,
69010,
]
redirect = set()
noredirect = set()
nometa = set()
cachedir = '/home/jtanner/.ansibullbot/cache'
gitrepo = GitRepoWrapper(cachedir=cachedir,
repo='https://github.com/ansible/ansible',
commit=None,
rebase=False)
rdata = gitrepo.get_file_content(u'.github/BOTMETA.yml')
botmeta = BotMetadataParser.parse_yaml(rdata)
cm = AnsibleComponentMatcher(cachedir=cachedir,
gitrepo=gitrepo,
botmeta=botmeta,
botmetafile=None,
email_cache=None,
usecache=True,
use_galaxy=True)
'''
mr = parse_match_results()
for issue in sorted(mr.keys(), key=lambda x: int(x.split('/')[-1]), reverse=True):
print(issue)
number = int(issue.split('/')[-1])
#if number != 68709:
# continue
print(number)
mfile = os.path.join('~/.ansibullbot/cache/ansible/ansible/issues/%s' % number, 'meta.json')
mfile = os.path.expanduser(mfile)
if os.path.exists(mfile):
with open(mfile, 'r') as f:
imeta = json.loads(f.read())
else:
nometa.add(issue)
imeta = {}
if imeta:
iw = MockIssueWrapper(issue, meta=imeta)
cfacts = get_collection_facts(iw, cm, imeta)
#pprint(cfacts)
if cfacts.get('needs_collection_redirect') == True:
redirect.add(issue)
else:
noredirect.add(issue)
#if not imeta['is_backport']:
# import epdb; epdb.st()
'''
mmap = {}
#gmatches = cm.search_ecosystem('contrib/inventory/ec2.py')
#import epdb; epdb.st()
mfiles = get_issues()
for mfile in mfiles:
with open(mfile, 'r') as f:
imeta = json.loads(f.read())
print(imeta['html_url'])
number = int(imeta['html_url'].split('/')[-1])
if number not in tocheck:
continue
newmeta = copy.deepcopy(imeta)
iw = MockIssueWrapper(imeta['html_url'], meta=newmeta, gitrepo=gitrepo)
#cmatches = cm.match_components(iw.title, iw.body, iw.component)
cmmeta = get_component_match_facts(iw, cm, [])
newmeta.update(cmmeta)
cfmeta = get_collection_facts(iw, cm, newmeta)
# check api deltas ...
#cm1 = cm.match(iw)
#cm2 = cm.match_components(iw.title, iw.body, iw.component, files=iw.files)
#import epdb; epdb.st()
print('component: %s' % iw.component)
print(cmmeta['component_filenames'])
#pprint(cfmeta)
cf2vals = [x for x in list(cfmeta['collection_filemap'].values()) if x]
cf1vals = [x for x in list(imeta['collection_filemap'].values()) if x]
'''
if cf1vals or cf2vals:
pprint(cf1vals)
pprint(cf2vals)
#import epdb; epdb.st()
'''
'''
if cf2vals != cf1vals:
pprint(cf1vals)
pprint(cf2vals)
import epdb; epdb.st()
'''
pprint(cfmeta)
import epdb
epdb.st()
print('# %s total issues|PRs without meta' % len(list(nometa)))
print('# %s total issues|PRs not redirected to collections' %
len(list(noredirect)))
print('# %s total issues|PRs redirected to collections' %
len(list(redirect)))
import epdb
epdb.st()