def test_owner_pr_submitter_is_maintainer_new_module(self): """ Submitter is a maintainer: pull request adds a new module: ensure owner_pr is False """ botmeta_files = {u'lib/ansible/modules/foo/bar.py': {u'maintainers': [u'ElsA', u'mscherer']}} CM = ComponentMatcherMock() CM.expected_results = [ { u'repo_filename': u'lib/ansible/modules/foo/bar.py', u'labels': [], u'support': None, u'maintainers': [u'ElsA', u'mscherer'], u'notify': [u'ElsA', u'mscherer'], u'ignore': [], } ] meta = self.meta.copy() datafile = u'tests/fixtures/shipit/0_issue.yml' statusfile = u'tests/fixtures/shipit/0_prstatus.json' with get_issue(datafile, statusfile) as iw: iw.pr_files = [MockFile(u'lib/ansible/modules/foo/bar.py')] iw.gitrepo = GitRepoWrapperMock() meta.update(get_component_match_facts(iw, CM, [])) facts = get_shipit_facts(iw, meta, botmeta_files, core_team=[u'bcoca'], botnames=[u'ansibot']) self.assertEqual(iw.submitter, u'mscherer') self.assertFalse(facts[u'owner_pr'])
def test_owner_pr_submitter_is_not_maintainer_of_all_updated_files(self): """ PR updates 2 files below module_utils, submitter is a maintainer from only one: ensure owner_pr isn't set """ BOTMETA = u""" --- macros: modules: lib/ansible/modules module_utils: lib/ansible/module_utils files: $module_utils/foo/bar.py: maintainers: ElsA Oliver $module_utils/baz/bar.py: maintainers: TiTi ZaZa """ module_indexer = create_indexer(textwrap.dedent(BOTMETA), {}) self.assertEqual(len(module_indexer.modules), 1) # ensure only fake data are loaded self.assertEqual(sorted(module_indexer.botmeta[u'files'][u'lib/ansible/module_utils/foo/bar.py'][u'maintainers']), [u'ElsA', u'Oliver']) self.assertEqual(sorted(module_indexer.botmeta[u'files'][u'lib/ansible/module_utils/baz/bar.py'][u'maintainers']), [u'TiTi', u'ZaZa']) CM = ComponentMatcherMock() CM.expected_results = [ { u'repo_filename': u'lib/ansible/module_utils/foo/bar.py', u'labels': [], u'support': None, u'maintainers': [u'ElsA', u'Oliver'], u'notify': [u'ElsA', u'Oliver'], u'ignore': [], }, { u'repo_filename': u'lib/ansible/modules/baz/bar.py', u'labels': [], u'support': None, u'maintainers': [u'TiTi', u'ZaZa'], u'notify': [u'TiTi', u'ZaZa'], u'ignore': [], } ] issue = IssueMock(u'/dev/null') issue.user.login = u'ElsA' issue.html_url = u'https://github.com/ansible/ansible/pull/123' iw = IssueWrapper(cachedir="", issue=issue) iw.pr_files = [ MockFile(u'lib/ansible/module_utils/foo/bar.py'), MockFile(u'lib/ansible/module_utils/baz/bar.py') ] iw.file_indexer = FileIndexerMock() iw.repo = MockRepo(repo_path='ansible/ansible') meta = self.meta.copy() iw._commits = [] meta.update(get_component_match_facts(iw, CM, [])) facts = get_shipit_facts(iw, meta, module_indexer, core_team=[u'bcoca', u'mscherer'], botnames=[u'ansibot']) self.assertEqual(iw.submitter, u'ElsA') self.assertFalse(facts[u'owner_pr'])
def test_owner_pr_submitter_is_maintainer_one_module_utils_file_updated(self): """ Submitter is a maintainer: ensure owner_pr is set (only one file below module_utils updated) """ botmeta_files = {'lib/ansible/module_utils/foo/bar.py': {'maintainers': ['ElsA', 'Oliver']}} datafile = u'tests/fixtures/shipit/2_issue.yml' statusfile = u'tests/fixtures/shipit/2_prstatus.json' with get_issue(datafile, statusfile) as iw: iw.pr_files = [MockFile(u'lib/ansible/module_utils/foo/bar.py')] # need to give the wrapper a list of known files to compare against iw.gitrepo = GitRepoWrapperMock() iw.gitrepo.files.append(u'lib/ansible/modules/foo/bar.py') # predefine what the matcher is going to return CM = ComponentMatcherMock() CM.expected_results = [ { u'repo_filename': u'lib/ansible/module_utils/foo/bar.py', u'labels': [], u'support': None, u'maintainers': [u'ElsA', u'Oliver'], u'notify': [u'ElsA', u'Oliver'], u'ignore': [], } ] meta = self.meta.copy() iw._commits = [] meta.update(get_component_match_facts(iw, CM, [])) facts = get_shipit_facts(iw, meta, botmeta_files, core_team=[u'bcoca', u'mscherer'], botnames=[u'ansibot']) self.assertEqual(iw.submitter, u'ElsA') self.assertTrue(facts[u'owner_pr'])
def test_owner_pr_module_utils_and_modules_updated_submitter_maintainer_2(self): """ PR updates 2 files (one below modules, the other below module_utils), submitter is a maintainer from both, check that owner_pr is set. Submitter is maintainer from module_utils file. """ BOTMETA = u""" --- macros: modules: lib/ansible/modules module_utils: lib/ansible/module_utils files: $modules/foo/bar.py: maintainers: ElsA ZaZa $module_utils/baz/bar.py: maintainers: TiTi mscherer """ modules = {u'lib/ansible/modules/foo/bar.py': None} module_indexer = create_indexer(textwrap.dedent(BOTMETA), modules) self.assertEqual(len(module_indexer.modules), 2) # ensure only fake data are loaded self.assertEqual(sorted(module_indexer.botmeta[u'files'][u'lib/ansible/modules/foo/bar.py'][u'maintainers']), [u'ElsA', u'ZaZa']) self.assertEqual(sorted(module_indexer.botmeta[u'files'][u'lib/ansible/module_utils/baz/bar.py'][u'maintainers']), [u'TiTi', u'mscherer']) CM = ComponentMatcherMock() CM.expected_results = [ { u'repo_filename': u'lib/ansible/module_utils/foo/bar.py', u'labels': [], u'support': None, u'maintainers': [u'ElsA', u'mscherer'], u'notify': [u'ElsA', u'mscherer'], u'ignore': [], }, { u'repo_filename': u'lib/ansible/modules/baz/bar.py', u'labels': [], u'support': None, u'maintainers': [u'TiTi', u'ZaZa'], u'notify': [u'TiTi', u'ZaZa'], u'ignore': [], } ] meta = self.meta.copy() datafile = u'tests/fixtures/shipit/0_issue.yml' statusfile = u'tests/fixtures/shipit/0_prstatus.json' with get_issue(datafile, statusfile) as iw: iw.pr_files = [ MockFile(u'lib/ansible/modules/foo/bar.py'), MockFile(u'lib/ansible/module_utils/baz/bar.py') ] iw.file_indexer = FileIndexerMock() meta.update(get_component_match_facts(iw, CM, [])) facts = get_shipit_facts(iw, meta, module_indexer, core_team=[u'bcoca'], botnames=[u'ansibot']) self.assertEqual(iw.submitter, u'mscherer') self.assertFalse(facts[u'owner_pr'])
def test_owner_pr_module_utils_and_modules_updated_submitter_maintainer_1( self): """ PR updates 2 files (one below modules, the other below module_utils), submitter is a maintainer from both, check that owner_pr is set. Submitter is maintainer from module file. """ botmeta_files = { 'lib/ansible/modules/foo/bar.py': { 'maintainers': ['ElsA', 'mscherer'] }, 'lib/ansible/module_utils/baz/bar.py': { 'maintainers': ['TiTi', 'ZaZa'] }, } CM = ComponentMatcherMock() CM.expected_results = [{ 'repo_filename': 'lib/ansible/module_utils/foo/bar.py', 'labels': [], 'support': None, 'maintainers': ['ElsA', 'mscherer'], 'notify': ['ElsA', 'mscherer'], 'ignore': [], }, { 'repo_filename': 'lib/ansible/modules/baz/bar.py', 'labels': [], 'support': None, 'maintainers': ['TiTi', 'ZaZa'], 'notify': ['TiTi', 'ZaZa'], 'ignore': [], }] meta = self.meta.copy() datafile = 'tests/fixtures/shipit/0_issue.yml' statusfile = 'tests/fixtures/shipit/0_prstatus.json' with get_issue(datafile, statusfile) as iw: iw.pr_files = [ MockFile('lib/ansible/modules/foo/bar.py'), MockFile('lib/ansible/module_utils/baz/bar.py') ] iw.gitrepo = GitRepoWrapperMock() iw.gitrepo.files = [ 'lib/ansible/modules/foo/bar.py', 'lib/ansible/module_utils/baz/bar.py' ] iw._commits = [] meta.update(get_component_match_facts(iw, CM, [])) facts = get_shipit_facts(iw, meta, botmeta_files, core_team=['bcoca', 'mscherer'], botnames=['ansibot']) self.assertEqual(iw.submitter, 'mscherer') self.assertFalse(facts['owner_pr'])
def test_owner_pr_submitter_is_maintainer_one_module_file_updated_changelog( self): """ Submitter is a maintainer: ensure owner_pr is set even if changelog fragment is present """ BOTMETA = u""" --- macros: modules: lib/ansible/modules files: $modules/foo/bar.py: maintainers: ElsA Oliver """ modules = {u'lib/ansible/modules/foo/bar.py': None} module_indexer = create_indexer(textwrap.dedent(BOTMETA), modules) self.assertEqual(len(module_indexer.modules), 2) # ensure only fake data are loaded self.assertEqual( sorted(module_indexer.botmeta[u'files'] [u'lib/ansible/modules/foo/bar.py'][u'maintainers']), [u'ElsA', u'Oliver']) datafile = u'tests/fixtures/shipit/2_issue.yml' statusfile = u'tests/fixtures/shipit/2_prstatus.json' with get_issue(datafile, statusfile) as iw: iw.pr_files = [ MockFile(u'lib/ansible/modules/foo/bar.py'), MockFile(u'changelogs/fragments/00000-fragment.yaml') ] # need to give the wrapper a list of known files to compare against iw.file_indexer = FileIndexerMock() iw.file_indexer.files.append(u'lib/ansible/modules/foo/bar.py') # predefine what the matcher is going to return CM = ComponentMatcherMock() CM.expected_results = [{ u'repo_filename': u'lib/ansible/modules/foo/bar.py', u'labels': [], u'support': None, u'maintainers': [u'ElsA', u'Oliver'], u'notify': [u'ElsA', u'Oliver'], u'ignore': [], }] meta = self.meta.copy() iw._commits = [] meta.update(get_component_match_facts(iw, CM, [])) facts = get_shipit_facts(iw, meta, module_indexer, core_team=[u'bcoca', u'mscherer'], botnames=[u'ansibot']) self.assertEqual(iw.submitter, u'ElsA') self.assertTrue(facts[u'owner_pr'])
def test_owner_pr_submitter_is_maintainer_one_module_utils_file_updated( self): """ Submitter is a maintainer: ensure owner_pr is set (only one file below module_utils updated) """ BOTMETA = """ --- macros: modules: lib/ansible/modules module_utils: lib/ansible/module_utils files: $module_utils/foo/bar.py: maintainers: ElsA Oliver """ modules = {'lib/ansible/module_utils/foo/bar.py': None} module_indexer = create_indexer(textwrap.dedent(BOTMETA), modules) self.assertEqual(len(module_indexer.modules), 2) # ensure only fake data are loaded self.assertEqual( sorted(module_indexer.botmeta['files'] ['lib/ansible/module_utils/foo/bar.py']['maintainers']), ['ElsA', 'Oliver']) issue = IssueMock('/dev/null') issue.user.login = '******' issue.html_url = 'https://github.com/ansible/ansible/pull/123' iw = IssueWrapper(cachedir="", issue=issue) iw.pr_files = [MockFile('lib/ansible/module_utils/foo/bar.py')] # need to give the wrapper a list of known files to compare against iw.file_indexer = FileIndexerMock() # predefine what the matcher is going to return CM = ComponentMatcherMock() CM.expected_results = [{ 'repo_filename': 'lib/ansible/module_utils/foo/bar.py', 'labels': [], 'support': None, 'maintainers': ['ElsA', 'Oliver'], 'notify': ['ElsA', 'Oliver'], 'ignore': [], }] meta = self.meta.copy() meta.update( get_component_match_facts(iw, meta, CM, FileIndexerMock(), module_indexer, [])) facts = get_shipit_facts(iw, meta, module_indexer, core_team=['bcoca', 'mscherer'], botnames=['ansibot']) self.assertEqual(iw.submitter, 'ElsA') self.assertTrue(facts['owner_pr'])
def test_owner_pr_submitter_is_maintainer_new_module(self): """ Submitter is a maintainer: pull request adds a new module: ensure owner_pr is False """ BOTMETA = """ --- macros: modules: lib/ansible/modules module_utils: lib/ansible/module_utils files: $modules/foo/bar.py: maintainers: ElsA mscherer """ modules = {} # new module module_indexer = create_indexer(textwrap.dedent(BOTMETA), modules) self.assertEqual(len(module_indexer.modules), 1) # ensure only fake data are loaded # Ensure that BOTMETA.yml updates doesn't interfere self.assertEqual( sorted(module_indexer.botmeta['files'] ['lib/ansible/modules/foo/bar.py']['maintainers']), ['ElsA', 'mscherer']) CM = ComponentMatcherMock() CM.expected_results = [{ 'repo_filename': 'lib/ansible/modules/foo/bar.py', 'labels': [], 'support': None, 'maintainers': ['ElsA', 'mscherer'], 'notify': ['ElsA', 'mscherer'], 'ignore': [], }] meta = self.meta.copy() datafile = 'tests/fixtures/shipit/0_issue.yml' statusfile = 'tests/fixtures/shipit/0_prstatus.json' with get_issue(datafile, statusfile) as iw: iw.pr_files = [MockFile('lib/ansible/modules/foo/bar.py')] iw.file_indexer = FileIndexerMock() #iw.file_indexer.files.append('lib/ansible/modules/foo/bar.py') meta.update( get_component_match_facts(iw, {}, CM, iw.file_indexer, module_indexer, [])) facts = get_shipit_facts(iw, meta, module_indexer, core_team=['bcoca'], botnames=['ansibot']) self.assertEqual(iw.submitter, 'mscherer') self.assertFalse(facts['owner_pr'])
def test_owner_pr_submitter_is_not_maintainer_of_all_updated_files(self): """ PR updates 2 files below module_utils, submitter is a maintainer from only one: ensure owner_pr isn't set """ botmeta_files = { 'lib/ansible/module_utils/foo/bar.py': { 'maintainers': ['ElsA', 'Oliver'] }, 'lib/ansible/module_utils/baz/bar.py': { 'maintainers': ['TiTi', 'ZaZa'] }, } CM = ComponentMatcherMock() CM.expected_results = [{ 'repo_filename': 'lib/ansible/module_utils/foo/bar.py', 'labels': [], 'support': None, 'maintainers': ['ElsA', 'Oliver'], 'notify': ['ElsA', 'Oliver'], 'ignore': [], }, { 'repo_filename': 'lib/ansible/modules/baz/bar.py', 'labels': [], 'support': None, 'maintainers': ['TiTi', 'ZaZa'], 'notify': ['TiTi', 'ZaZa'], 'ignore': [], }] issue = IssueMock('/dev/null') issue.user.login = '******' issue.html_url = 'https://github.com/ansible/ansible/pull/123' cachedir = tempfile.mkdtemp() gh = GithubWrapperMock() iw = IssueWrapper(cachedir=cachedir, issue=issue, github=gh) iw.pr_files = [ MockFile('lib/ansible/module_utils/foo/bar.py'), MockFile('lib/ansible/module_utils/baz/bar.py') ] iw.gitrepo = GitRepoWrapperMock() iw.repo = MockRepo(repo_path='ansible/ansible') meta = self.meta.copy() iw._commits = [] meta.update(get_component_match_facts(iw, CM, [])) facts = get_shipit_facts(iw, meta, botmeta_files, core_team=['bcoca', 'mscherer'], botnames=['ansibot']) shutil.rmtree(cachedir) self.assertEqual(iw.submitter, 'ElsA') self.assertFalse(facts['owner_pr'])
def test_owner_pr_module_utils_and_modules_updated_submitter_maintainer_2( self): """ PR updates 2 files (one below modules, the other below module_utils), submitter is a maintainer from both, check that owner_pr is set. Submitter is maintainer from module_utils file. """ BOTMETA = """ --- macros: modules: lib/ansible/modules module_utils: lib/ansible/module_utils files: $modules/foo/bar.py: maintainers: ElsA ZaZa $module_utils/baz/bar.py: maintainers: TiTi mscherer """ modules = {'lib/ansible/modules/foo/bar.py': None} module_indexer = create_indexer(textwrap.dedent(BOTMETA), modules) self.assertEqual(len(module_indexer.modules), 2) # ensure only fake data are loaded self.assertEqual( sorted(module_indexer.botmeta['files'] ['lib/ansible/modules/foo/bar.py']['maintainers']), ['ElsA', 'ZaZa']) self.assertEqual( sorted(module_indexer.botmeta['files'] ['lib/ansible/module_utils/baz/bar.py']['maintainers']), ['TiTi', 'mscherer']) meta = self.meta.copy() datafile = 'tests/fixtures/shipit/0_issue.yml' statusfile = 'tests/fixtures/shipit/0_prstatus.json' with get_issue(datafile, statusfile) as iw: iw.pr_files = [ MockFile('lib/ansible/modules/foo/bar.py'), MockFile('lib/ansible/module_utils/baz/bar.py') ] meta.update( get_component_match_facts(iw, {}, FileIndexerMock(), module_indexer, [])) facts = get_shipit_facts(iw, meta, module_indexer, core_team=['bcoca'], botnames=['ansibot']) self.assertEqual(iw.submitter, 'mscherer') self.assertFalse(facts['owner_pr'])
def test_review_facts_are_defined_module_utils(self): BOTMETA = u""" --- macros: modules: lib/ansible/modules module_utils: lib/ansible/module_utils files: $module_utils: support: community $modules/foo/bar.py: maintainers: ElsA ZaZa $module_utils/baz/bar.py: maintainers: TiTi mscherer """ modules = {u'lib/ansible/modules/foo/bar.py': None} module_indexer = create_indexer(textwrap.dedent(BOTMETA), modules) self.assertEqual(len(module_indexer.modules), 2) # ensure only fake data are loaded self.assertEqual(sorted(module_indexer.botmeta[u'files'][u'lib/ansible/modules/foo/bar.py'][u'maintainers']),[u'ElsA', u'ZaZa']) self.assertEqual(sorted(module_indexer.botmeta[u'files'][u'lib/ansible/module_utils/baz/bar.py'][u'maintainers']),[u'TiTi', u'mscherer']) datafile = u'tests/fixtures/shipit/2_issue.yml' statusfile = u'tests/fixtures/shipit/2_prstatus.json' with get_issue(datafile, statusfile) as iw: iw.pr_files = [MockFile(u'lib/ansible/module_utils/foo/bar.py')] # need to give the wrapper a list of known files to compare against iw.file_indexer = FileIndexerMock() iw.file_indexer.files.append(u'lib/ansible/modules/foo/bar.py') # predefine what the matcher is going to return CM = ComponentMatcherMock() CM.expected_results = [ { u'repo_filename': u'lib/ansible/module_utils/foo/bar.py', u'labels': [], u'support': None, u'maintainers': [u'ElsA', u'Oliver'], u'notify': [u'ElsA', u'Oliver'], u'ignore': [], } ] meta = self.meta.copy() iw._commits = [] meta.update(get_component_match_facts(iw, CM, [])) meta.update(get_shipit_facts(iw, meta, module_indexer, core_team=[u'bcoca'], botnames=[u'ansibot'])) facts = get_review_facts(iw, meta) self.assertTrue(facts[u'community_review']) self.assertFalse(facts[u'core_review']) self.assertFalse(facts[u'committer_review'])
def run(self): '''Emit an html report of each file and it's metadata''' component_facts = {} lblacklist = [ 'ansible', ] lblacklist = [] filenames = sorted(self.gitrepo.files) filenames = [x for x in filenames if not x.startswith('/')] filenames = [x for x in filenames if not '__pycache__' in x] filenames = [x for x in filenames if not x.endswith('.pyo')] filenames = [x for x in filenames if not x.endswith('.pyo')] for fn in filenames: logging.debug(fn) meta = self.component_matcher.search_by_filepath(fn) iw = IssueMock() iw._component = fn try: facts = get_component_match_facts(iw, self.component_matcher, []) component_facts[fn] = {} component_facts[fn]['component'] = facts['component_name'] component_facts[fn]['support'] = facts['component_matches'][0][ 'support'] labels = facts['component_matches'][0]['labels'][:] labels += facts['component_labels'] labels = sorted(set(labels)) labels = [x for x in labels if x not in lblacklist] component_facts[fn]['labels'] = ','.join(labels) component_facts[fn]['maintainers'] = \ ','.join(facts['component_matches'][0]['maintainers']) except Exception as e: component_facts[fn] = {'error': '%s' % e} logging.error(fn) logging.error(e) logging.info('done matching') logging.info('building report in %s' % self.args.dest) dest = os.path.expanduser(self.args.dest) if os.path.exists(dest): shutil.rmtree(dest) skel = os.path.join(TEMPLATES, 'metareport') shutil.copytree(skel, dest) with open(os.path.join(dest, 'data.json'), 'w') as f: f.write(json.dumps(list(component_facts.values())))
def test_owner_pr_submitter_is_not_maintainer_of_all_updated_files(self): """ PR updates 2 files below module_utils, submitter is a maintainer from only one: ensure owner_pr isn't set """ BOTMETA = """ --- macros: modules: lib/ansible/modules module_utils: lib/ansible/module_utils files: $module_utils/foo/bar.py: maintainers: ElsA Oliver $module_utils/baz/bar.py: maintainers: TiTi ZaZa """ module_indexer = create_indexer(textwrap.dedent(BOTMETA), {}) self.assertEqual(len(module_indexer.modules), 1) # ensure only fake data are loaded self.assertEqual( sorted(module_indexer.botmeta['files'] ['lib/ansible/module_utils/foo/bar.py']['maintainers']), ['ElsA', 'Oliver']) self.assertEqual( sorted(module_indexer.botmeta['files'] ['lib/ansible/module_utils/baz/bar.py']['maintainers']), ['TiTi', 'ZaZa']) issue = IssueMock('/dev/null') issue.user.login = '******' issue.html_url = 'https://github.com/ansible/ansible/pull/123' iw = IssueWrapper(cachedir="", issue=issue) iw.pr_files = [ MockFile('lib/ansible/module_utils/foo/bar.py'), MockFile('lib/ansible/module_utils/baz/bar.py') ] meta = self.meta.copy() meta.update( get_component_match_facts(iw, {}, FileIndexerMock(), module_indexer, [])) facts = get_shipit_facts(iw, meta, module_indexer, core_team=['bcoca', 'mscherer'], botnames=['ansibot']) self.assertEqual(iw.submitter, 'ElsA') self.assertFalse(facts['owner_pr'])
def test_owner_pr_submitter_is_maintainer_new_module(self): """ Submitter is a maintainer: pull request adds a new module: ensure owner_pr is False """ BOTMETA = u""" --- macros: modules: lib/ansible/modules module_utils: lib/ansible/module_utils files: $modules/foo/bar.py: maintainers: ElsA mscherer """ modules = {} # new module module_indexer = create_indexer(textwrap.dedent(BOTMETA), modules) self.assertEqual(len(module_indexer.modules), 1) # ensure only fake data are loaded # Ensure that BOTMETA.yml updates doesn't interfere self.assertEqual(sorted(module_indexer.botmeta[u'files'][u'lib/ansible/modules/foo/bar.py'][u'maintainers']), [u'ElsA', u'mscherer']) CM = ComponentMatcherMock() CM.expected_results = [ { u'repo_filename': u'lib/ansible/modules/foo/bar.py', u'labels': [], u'support': None, u'maintainers': [u'ElsA', u'mscherer'], u'notify': [u'ElsA', u'mscherer'], u'ignore': [], } ] meta = self.meta.copy() datafile = u'tests/fixtures/shipit/0_issue.yml' statusfile = u'tests/fixtures/shipit/0_prstatus.json' with get_issue(datafile, statusfile) as iw: iw.pr_files = [MockFile(u'lib/ansible/modules/foo/bar.py')] iw.file_indexer = FileIndexerMock() meta.update(get_component_match_facts(iw, CM, [])) facts = get_shipit_facts(iw, meta, module_indexer, core_team=[u'bcoca'], botnames=[u'ansibot']) self.assertEqual(iw.submitter, u'mscherer') self.assertFalse(facts[u'owner_pr'])
def test_review_facts_are_defined_module_utils(self): botmeta_files = { 'lib/ansible/module_utils': { 'support': 'community' }, 'lib/ansible/modules/foo/bar.py': { 'maintainers': ['ElsA', 'ZaZa'] }, 'lib/ansible/module_utils/baz/bar.py': { 'maintainers': ['TiTi', 'mscherer'] }, } datafile = 'tests/fixtures/shipit/2_issue.yml' statusfile = 'tests/fixtures/shipit/2_prstatus.json' with get_issue(datafile, statusfile) as iw: iw.pr_files = [MockFile('lib/ansible/module_utils/foo/bar.py')] # need to give the wrapper a list of known files to compare against iw.gitrepo = GitRepoWrapperMock() iw.gitrepo.files.append('lib/ansible/modules/foo/bar.py') # predefine what the matcher is going to return CM = ComponentMatcherMock() CM.expected_results = [{ 'repo_filename': 'lib/ansible/module_utils/foo/bar.py', 'labels': [], 'support': None, 'maintainers': ['ElsA', 'Oliver'], 'notify': ['ElsA', 'Oliver'], 'ignore': [], }] meta = self.meta.copy() iw._commits = [] meta.update(get_component_match_facts(iw, CM, [])) meta.update( get_shipit_facts(iw, meta, botmeta_files, core_team=['bcoca'], botnames=['ansibot'])) facts = get_review_facts(iw, meta) self.assertTrue(facts['community_review']) self.assertFalse(facts['core_review']) self.assertFalse(facts['committer_review'])
def test_owner_pr_submitter_is_maintainer_one_module_utils_file_updated( self): """ Submitter is a maintainer: ensure owner_pr is set (only one file below module_utils updated) """ BOTMETA = """ --- macros: modules: lib/ansible/modules module_utils: lib/ansible/module_utils files: $module_utils/foo/bar.py: maintainers: ElsA Oliver """ modules = {'lib/ansible/module_utils/foo/bar.py': None} module_indexer = create_indexer(textwrap.dedent(BOTMETA), modules) self.assertEqual(len(module_indexer.modules), 2) # ensure only fake data are loaded self.assertEqual( sorted(module_indexer.botmeta['files'] ['lib/ansible/module_utils/foo/bar.py']['maintainers']), ['ElsA', 'Oliver']) issue = IssueMock('/dev/null') issue.user.login = '******' issue.html_url = 'https://github.com/ansible/ansible/pull/123' iw = IssueWrapper(cachedir="", issue=issue) iw.pr_files = [MockFile('lib/ansible/module_utils/foo/bar.py')] meta = self.meta.copy() meta.update( get_component_match_facts(iw, None, FileIndexerMock(), module_indexer, [])) facts = get_shipit_facts(iw, meta, module_indexer, core_team=['bcoca', 'mscherer'], botnames=['ansibot']) self.assertEqual(iw.submitter, 'ElsA') self.assertTrue(facts['owner_pr'])
def test_owner_pr_submitter_is_maintainer_one_module_file_updated_changelog( self): """ Submitter is a maintainer: ensure owner_pr is set even if changelog fragment is present """ botmeta_files = { 'lib/ansible/modules/foo/bar.py': { 'maintainers': ['ElsA', 'Oliver'] } } datafile = 'tests/fixtures/shipit/2_issue.yml' statusfile = 'tests/fixtures/shipit/2_prstatus.json' with get_issue(datafile, statusfile) as iw: iw._pr_files = [ MockFile('lib/ansible/modules/foo/bar.py'), MockFile('changelogs/fragments/00000-fragment.yaml') ] # need to give the wrapper a list of known files to compare against iw.gitrepo = GitRepoWrapperMock() iw.gitrepo.files.append('lib/ansible/modules/foo/bar.py') # predefine what the matcher is going to return CM = ComponentMatcherMock() CM.expected_results = [{ 'repo_filename': 'lib/ansible/modules/foo/bar.py', 'labels': [], 'support': None, 'maintainers': ['ElsA', 'Oliver'], 'notify': ['ElsA', 'Oliver'], 'ignore': [], }] meta = self.meta.copy() iw._commits = [] meta.update(get_component_match_facts(iw, CM, [])) facts = get_shipit_facts(iw, meta, botmeta_files, core_team=['bcoca', 'mscherer'], botnames=['ansibot']) self.assertEqual(iw.submitter, 'ElsA') self.assertTrue(facts['owner_pr'])
def test_owner_pr_submitter_is_maintainer_one_modules_file_updated(self): """ Submitter is a maintainer: ensure owner_pr is set (only one file below modules updated) """ BOTMETA = """ --- macros: modules: lib/ansible/modules module_utils: lib/ansible/module_utils files: $modules/foo/bar.py: maintainers: ElsA mscherer """ modules = {'lib/ansible/modules/foo/bar.py': None} module_indexer = create_indexer(textwrap.dedent(BOTMETA), modules) self.assertEqual(len(module_indexer.modules), 2) # ensure only fake data are loaded self.assertEqual( sorted(module_indexer.botmeta['files'] ['lib/ansible/modules/foo/bar.py']['maintainers']), ['ElsA', 'mscherer']) meta = self.meta.copy() datafile = 'tests/fixtures/shipit/0_issue.yml' statusfile = 'tests/fixtures/shipit/0_prstatus.json' with get_issue(datafile, statusfile) as iw: iw.pr_files = [MockFile('lib/ansible/modules/foo/bar.py')] meta.update( get_component_match_facts(iw, {}, FileIndexerMock(), module_indexer, [])) facts = get_shipit_facts(iw, meta, module_indexer, core_team=['bcoca'], botnames=['ansibot']) self.assertEqual(iw.submitter, 'mscherer') self.assertTrue(facts['owner_pr'])
def test_owner_pr_submitter_is_maintainer_one_modules_file_updated(self): """ Submitter is a maintainer: ensure owner_pr is set (only one file below modules updated) """ botmeta_files = { 'lib/ansible/modules/foo/bar.py': { 'maintainers': ['ElsA', 'mscherer'] } } CM = ComponentMatcherMock() CM.expected_results = [{ 'repo_filename': 'lib/ansible/modules/foo/bar.py', 'labels': [], 'support': None, 'maintainers': ['ElsA', 'mscherer'], 'notify': ['ElsA', 'mscherer'], 'ignore': [], }] meta = self.meta.copy() datafile = 'tests/fixtures/shipit/0_issue.yml' statusfile = 'tests/fixtures/shipit/0_prstatus.json' with get_issue(datafile, statusfile) as iw: iw.pr_files = [MockFile('lib/ansible/modules/foo/bar.py')] iw.gitrepo = GitRepoWrapperMock() iw.gitrepo.files.append('lib/ansible/modules/foo/bar.py') meta.update(get_component_match_facts(iw, CM, [])) facts = get_shipit_facts(iw, meta, botmeta_files, core_team=['bcoca'], botnames=['ansibot']) self.assertEqual(iw.submitter, 'mscherer') self.assertTrue(facts['owner_pr'])
def main(): set_logger() METAFILES = extract_metafiles() SKIP = load_skip() EXPECTED = load_expected() MATCH_MAP = load_match_map() ERRORS = [] ERRORS_COMPONENTS = [] start_at = None if len(sys.argv) == 2: start_at = int(sys.argv[1]) FI = FileIndexer(checkoutdir=CACHEDIR) with open('/tmp/files.json', 'wb') as f: f.write(json.dumps(FI.files, indent=2)) GQLC = GithubGraphQLClient(C.DEFAULT_GITHUB_TOKEN) MI = ModuleIndexer(cachedir=CACHEDIR, gh_client=GQLC, blames=False, commits=False) CM = AnsibleComponentMatcher(cachedir=CACHEDIR) for k, v in MI.modules.items(): if k in MATCH_MAP: MATCH_MAP.pop(k, None) kname = v.get('name') if kname not in MATCH_MAP: MATCH_MAP[kname] = v.get('repo_filename') if kname + ' module' not in MATCH_MAP: MATCH_MAP[kname + ' module'] = v.get('repo_filename') if kname + 'module: ' + kname not in MATCH_MAP: MATCH_MAP['module: ' + kname] = v.get('repo_filename') if kname + 'module ' + kname not in MATCH_MAP: MATCH_MAP['module ' + kname] = v.get('repo_filename') # /modules/remote_management/foreman/katello.py pname = k.replace('lib/ansible', '') if pname not in MATCH_MAP: MATCH_MAP[pname] = v.get('repo_filename') # ansible/modules/packaging/os/rpm_key.py pname = k.replace('lib/', '/') if pname not in MATCH_MAP: MATCH_MAP[pname] = v.get('repo_filename') # /ansible/modules/packaging/os/rpm_key.py pname = k.replace('lib/', '') if pname not in MATCH_MAP: MATCH_MAP[pname] = v.get('repo_filename') # ansible/lib/ansible/modules/monitoring/monit.py pname = 'ansible/' + k if pname not in MATCH_MAP: MATCH_MAP[pname] = v.get('repo_filename') # network/f5/bigip_gtm_wide_ip pname = k.replace('lib/ansible/modules/', '') pname = pname.replace('.py', '') pname = pname.replace('.ps1', '') if pname not in MATCH_MAP: MATCH_MAP[pname] = v.get('repo_filename') # network/f5/bigip_gtm_wide_ip.py pname = k.replace('lib/ansible/modules/', '') if pname not in MATCH_MAP: MATCH_MAP[pname] = v.get('repo_filename') # modules/packaging/os/pkgng.py pname = k.replace('lib/ansible/', '') if pname not in MATCH_MAP: MATCH_MAP[pname] = v.get('repo_filename') save_match_map(MATCH_MAP) total = len(METAFILES) for IDMF, MF in enumerate(METAFILES): if start_at and IDMF < start_at: continue with open(MF, 'rb') as f: meta = json.loads(f.read()) if not meta.get('is_issue'): continue component = meta.get('template_data', {}).get('component_raw') #if component != 'Module `synchronize`': #if component != 'Module: include_role': # continue if component: print(f'------------------------------------------ {total}|{IDMF}') print(meta['html_url']) print(meta['title']) print(component) hurl = meta['html_url'] if hurl in SKIP: continue # bad template or bad template parsing if len(component) > 100: continue iw = IssueWrapperMock(meta) if 'module' not in iw.body.lower( ) and 'module' not in iw.title.lower(): continue expected_fns = [] # OLD METHOD if hurl not in EXPECTED and component not in MATCH_MAP: cmf = get_component_match_facts(iw, meta, FI, MI, LABELS) expected_fns = cmf.get('module_match') if not isinstance(expected_fns, list): expected_fns = [expected_fns] expected_fns = [x['repo_filename'] for x in expected_fns if x] if 'component_matches' in cmf: expected_fns = [ x['filename'] for x in cmf['component_matches'] ] expected_fns = sorted(set(expected_fns)) # NEW METHOD cmr = CM.match_components(iw.title, iw.body, iw.template_data.get('component_raw')) cmr_fns = [x['repo_filename'] for x in cmr if x] cmr_fns = sorted(set(cmr_fns)) # VALIDATE FROM EXPECTED IF KNOWN if hurl in EXPECTED: if EXPECTED[hurl] and not isinstance(EXPECTED[hurl], list): expected_fns = [EXPECTED[hurl]] elif EXPECTED[hurl]: expected_fns = EXPECTED[hurl] else: expected_fns = [] # USE THE CACHED MAP if component in MATCH_MAP: expected_fns = MATCH_MAP[component] if not isinstance(expected_fns, list): expected_fns = [expected_fns] elif component.lower() in MATCH_MAP: expected_fns = MATCH_MAP[component.lower()] if not isinstance(expected_fns, list): expected_fns = [expected_fns] elif component.startswith(':\n') and component.endswith(' module'): mapkey = component.lstrip(':\n') if mapkey in MATCH_MAP: expected_fns = MATCH_MAP[mapkey] if not isinstance(expected_fns, list): expected_fns = [expected_fns] # OLD CODE USED ACTION PLUGINS INSTEAD OF MODULES if expected_fns != cmr_fns and hurl not in EXPECTED: if len(expected_fns) == 1 and len( cmr_fns) == 1 and 'plugins/action' in expected_fns[0]: e_bn = os.path.basename(expected_fns[0]) c_bn = os.path.basename(cmr_fns[0]) if e_bn == c_bn: MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue # DOCS URLS if expected_fns != cmr_fns and hurl not in EXPECTED: if len(cmr_fns) == 1 and 'lib/ansible/modules' in cmr_fns[0]: c_bn = os.path.basename(cmr_fns[0]) if f'docs.ansible.com/ansible/latest/{c_bn}_module.html' in component: MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue elif CM.strategy in ['search_by_regex_urls']: MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue # NXOS ISSUES HAVE NXOS_VERSION HEADER if '- nxos' in component: if len(cmr_fns) == 1: if os.path.basename(cmr_fns[0]).replace('.py', '') in component: MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue #import epdb; epdb.st() # ODDBALL MODULE COMPONENTS if len(cmr_fns) == 1 and 'lib/ansible/modules' in cmr_fns[0]: bn = os.path.basename(cmr_fns[0]) bn = bn.replace('.py', '') bn = bn.replace('.ps1', '') if (bn in component or bn.lstrip('_') in component) and 'module' in component.lower(): MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue elif component == '- ' + bn: MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue elif component == bn + '.py' or component == bn + '.ps1': MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue elif component == '_' + bn + '.py' or component == '_' + bn + '.ps1': MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue elif component == ':\n' + bn or component == ':\n' + bn.lstrip( '_'): MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue # 'multiple modules', etc ... if component in CM.KEYWORDS or component.lower() in CM.KEYWORDS: if component in CM.KEYWORDS and CM.KEYWORDS[ component] is None and not cmr_fns: MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue elif component.lower() in CM.KEYWORDS and CM.KEYWORDS[ component.lower()] is None and not cmr_fns: MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue elif len(cmr_fns) == 1 and cmr_fns[0] == CM.KEYWORDS.get( component): MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue elif len(cmr_fns) == 1 and cmr_fns[0] == CM.KEYWORDS.get( component.lower()): MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue if component.lstrip('-').strip() in CM.KEYWORDS and len( cmr_fns) == 1: cname = component.lstrip('-').strip() if CM.KEYWORDS[cname] == cmr_fns[0]: MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue if component.endswith(' lookup') and len( cmr_fns ) == 1 and 'lib/ansible/plugins/lookup' in cmr_fns[0]: MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue if component.endswith(' inventory script') and len( cmr_fns) == 1 and 'contrib/inventory' in cmr_fns[0]: MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue if component.startswith('ansible/lib') and len(cmr_fns) == 1: fn = cmr_fns[0] if 'ansible/' + fn == component: MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue if component.endswith(' inventory plugin') and len(cmr_fns) == 1: fn = cmr_fns[0] if fn.startswith('lib/ansible/plugins/inventory'): MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue if component == 'ec2.py' and cmr_fns and 'contrib/inventory/ec2.py' in cmr_fns: MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue if len(expected_fns) == 1 and len(cmr_fns) == 1: if os.path.basename(expected_fns[0]) == os.path.basename( cmr_fns[0]): MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue # COMPARE AND RECORD if expected_fns != cmr_fns and hurl not in EXPECTED: if component in MATCH_MAP or component.lower() in MATCH_MAP: if component.lower() in MATCH_MAP: mmc = MATCH_MAP[component.lower()] else: mmc = MATCH_MAP[component] if not isinstance(mmc, list): mmc == [mmc] if mmc == cmr_fns: EXPECTED[iw.html_url] = cmr_fns save_expected(EXPECTED) continue print('## COMPONENT ...') print(component) print('## EXPECTED ...') pprint(expected_fns) print('## RESULT ...') pprint(cmr_fns) print('## STRATEGIES ..') pprint(CM.strategy) pprint(CM.strategies) print('--------------------------------') res = raw_input('Is the result correct? (y/n/s/d): ') if res.lower() in ['y', 'yes']: MATCH_MAP[component] = cmr_fns EXPECTED[iw.html_url] = cmr_fns save_expected(EXPECTED) continue elif res.lower() in ['s', 'skip']: SKIP.append(hurl) save_skip(SKIP) continue elif res.lower() in ['d', 'debug']: import epdb epdb.st() ERRORS.append(iw.html_url) ERRORS_COMPONENTS.append({ 'url': iw.html_url, 'component': component, 'component_raw': iw.template_data.get('component_raw'), 'result': cmr_fns, 'expected': expected_fns, 'strategy': CM.strategy, 'strategies': CM.strategies }) else: if component not in MATCH_MAP: MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) if hurl not in EXPECTED: EXPECTED[hurl] = cmr_fns save_expected(EXPECTED) continue pprint(ERRORS) fn = os.path.join(FIXTUREDIR, 'component_errors.json') with open(fn, 'wb') as f: f.write(json.dumps(ERRORS_COMPONENTS, indent=2, sort_keys=True)) clean_metafiles(METAFILES)
def main(): set_logger() METAFILES = extract_metafiles() SKIP = load_skip() EXPECTED = load_expected() MATCH_MAP = load_match_map() ERRORS = [] ERRORS_COMPONENTS = [] start_at = None if len(sys.argv) == 2: start_at = int(sys.argv[1]) FI = FileIndexer(checkoutdir=CACHEDIR) with open('/tmp/files.json', 'wb') as f: f.write(json.dumps(FI.files, indent=2)) GQLC = GithubGraphQLClient(C.DEFAULT_GITHUB_TOKEN) #GWS = GithubWebScraper(cachedir=CACHEDIR) MI = ModuleIndexer(cachedir=CACHEDIR, gh_client=GQLC, blames=False, commits=False) #CM = ComponentMatcher(cachedir=CACHEDIR, module_indexer=MI, file_indexer=FI) CM = AnsibleComponentMatcher(cachedir=CACHEDIR) for k,v in MI.modules.items(): if k in MATCH_MAP: MATCH_MAP.pop(k, None) kname = v.get('name') if kname not in MATCH_MAP: MATCH_MAP[kname] = v.get('repo_filename') if kname + ' module' not in MATCH_MAP: MATCH_MAP[kname + ' module'] = v.get('repo_filename') if kname + 'module: ' + kname not in MATCH_MAP: MATCH_MAP['module: ' + kname] = v.get('repo_filename') if kname + 'module ' + kname not in MATCH_MAP: MATCH_MAP['module ' + kname] = v.get('repo_filename') # /modules/remote_management/foreman/katello.py pname = k.replace('lib/ansible', '') if pname not in MATCH_MAP: MATCH_MAP[pname] = v.get('repo_filename') # ansible/modules/packaging/os/rpm_key.py pname = k.replace('lib/', '/') if pname not in MATCH_MAP: MATCH_MAP[pname] = v.get('repo_filename') # /ansible/modules/packaging/os/rpm_key.py pname = k.replace('lib/', '') if pname not in MATCH_MAP: MATCH_MAP[pname] = v.get('repo_filename') # ansible/lib/ansible/modules/monitoring/monit.py pname = 'ansible/' + k if pname not in MATCH_MAP: MATCH_MAP[pname] = v.get('repo_filename') # network/f5/bigip_gtm_wide_ip pname = k.replace('lib/ansible/modules/', '') pname = pname.replace('.py', '') pname = pname.replace('.ps1', '') if pname not in MATCH_MAP: MATCH_MAP[pname] = v.get('repo_filename') # network/f5/bigip_gtm_wide_ip.py pname = k.replace('lib/ansible/modules/', '') if pname not in MATCH_MAP: MATCH_MAP[pname] = v.get('repo_filename') # modules/packaging/os/pkgng.py pname = k.replace('lib/ansible/', '') if pname not in MATCH_MAP: MATCH_MAP[pname] = v.get('repo_filename') save_match_map(MATCH_MAP) total = len(METAFILES) for IDMF,MF in enumerate(METAFILES): if start_at and IDMF < start_at: continue with open(MF, 'rb') as f: meta = json.loads(f.read()) if not meta.get('is_issue'): continue component = meta.get('template_data', {}).get('component_raw') #if component != 'Module `synchronize`': #if component != 'Module: include_role': # continue if component: print('------------------------------------------ {}|{}'.format(total, IDMF)) print(meta['html_url']) print(meta['title']) print(component) hurl = meta['html_url'] if hurl in SKIP: continue # bad template or bad template parsing if len(component) > 100: continue iw = IssueWrapperMock(meta) if 'module' not in iw.body.lower() and 'module' not in iw.title.lower(): continue expected_fns = [] # OLD METHOD if hurl not in EXPECTED and component not in MATCH_MAP: cmf = get_component_match_facts(iw, meta, FI, MI, LABELS) expected_fns = cmf.get('module_match') if not isinstance(expected_fns, list): expected_fns = [expected_fns] expected_fns = [x['repo_filename'] for x in expected_fns if x] if 'component_matches' in cmf: expected_fns = [x['filename'] for x in cmf['component_matches']] expected_fns = sorted(set(expected_fns)) # NEW METHOD cmr = CM.match_components(iw.title, iw.body, iw.template_data.get('component_raw')) cmr_fns = [x['repo_filename'] for x in cmr if x] cmr_fns = sorted(set(cmr_fns)) # VALIDATE FROM EXPECTED IF KNOWN if hurl in EXPECTED: if EXPECTED[hurl] and not isinstance(EXPECTED[hurl], list): expected_fns = [EXPECTED[hurl]] elif EXPECTED[hurl]: expected_fns = EXPECTED[hurl] else: expected_fns = [] # USE THE CACHED MAP if component in MATCH_MAP: expected_fns = MATCH_MAP[component] if not isinstance(expected_fns, list): expected_fns = [expected_fns] elif component.lower() in MATCH_MAP: expected_fns = MATCH_MAP[component.lower()] if not isinstance(expected_fns, list): expected_fns = [expected_fns] elif component.startswith(':\n') and component.endswith(' module'): mapkey = component.lstrip(':\n') if mapkey in MATCH_MAP: expected_fns = MATCH_MAP[mapkey] if not isinstance(expected_fns, list): expected_fns = [expected_fns] # OLD CODE USED ACTION PLUGINS INSTEAD OF MODULES if expected_fns != cmr_fns and hurl not in EXPECTED: if len(expected_fns) == 1 and len(cmr_fns) == 1 and 'plugins/action' in expected_fns[0]: e_bn = os.path.basename(expected_fns[0]) c_bn = os.path.basename(cmr_fns[0]) if e_bn == c_bn: MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue # DOCS URLS if expected_fns != cmr_fns and hurl not in EXPECTED: if len(cmr_fns) == 1 and 'lib/ansible/modules' in cmr_fns[0]: c_bn = os.path.basename(cmr_fns[0]) if 'docs.ansible.com/ansible/latest/{}_module.html'.format(c_bn) in component: MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue elif CM.strategy in ['search_by_regex_urls']: MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue # NXOS ISSUES HAVE NXOS_VERSION HEADER if '- nxos' in component: if len(cmr_fns) == 1: if os.path.basename(cmr_fns[0]).replace('.py', '') in component: MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue #import epdb; epdb.st() # ODDBALL MODULE COMPONENTS if len(cmr_fns) == 1 and 'lib/ansible/modules' in cmr_fns[0]: bn = os.path.basename(cmr_fns[0]) bn = bn.replace('.py', '') bn = bn.replace('.ps1', '') if (bn in component or bn.lstrip('_') in component) and 'module' in component.lower(): MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue elif component == '- ' + bn: MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue elif component == bn + '.py' or component == bn + '.ps1': MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue elif component == '_' + bn + '.py' or component == '_' + bn + '.ps1': MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue elif component == ':\n' + bn or component == ':\n' + bn.lstrip('_'): MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue # 'multiple modules', etc ... if component in CM.KEYWORDS or component.lower() in CM.KEYWORDS: if component in CM.KEYWORDS and CM.KEYWORDS[component] is None and not cmr_fns: MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue elif component.lower() in CM.KEYWORDS and CM.KEYWORDS[component.lower()] is None and not cmr_fns: MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue elif len(cmr_fns) == 1 and cmr_fns[0] == CM.KEYWORDS.get(component): MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue elif len(cmr_fns) == 1 and cmr_fns[0] == CM.KEYWORDS.get(component.lower()): MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue if component.lstrip('-').strip() in CM.KEYWORDS and len(cmr_fns) == 1: cname = component.lstrip('-').strip() if CM.KEYWORDS[cname] == cmr_fns[0]: MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue if component.endswith(' lookup') and len(cmr_fns) == 1 and 'lib/ansible/plugins/lookup' in cmr_fns[0]: MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue if component.endswith(' inventory script') and len(cmr_fns) == 1 and 'contrib/inventory' in cmr_fns[0]: MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue if component.startswith('ansible/lib') and len(cmr_fns) == 1: fn = cmr_fns[0] if 'ansible/' + fn == component: MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue if component.endswith(' inventory plugin') and len(cmr_fns) == 1: fn = cmr_fns[0] if fn.startswith('lib/ansible/plugins/inventory'): MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue if component == 'ec2.py' and cmr_fns and 'contrib/inventory/ec2.py' in cmr_fns: MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue if len(expected_fns) == 1 and len(cmr_fns) == 1: if os.path.basename(expected_fns[0]) == os.path.basename(cmr_fns[0]): MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) continue # COMPARE AND RECORD if expected_fns != cmr_fns and hurl not in EXPECTED: if component in MATCH_MAP or component.lower() in MATCH_MAP: if component.lower() in MATCH_MAP: mmc = MATCH_MAP[component.lower()] else: mmc = MATCH_MAP[component] if not isinstance(mmc, list): mmc == [mmc] if mmc == cmr_fns: EXPECTED[iw.html_url] = cmr_fns save_expected(EXPECTED) continue print('## COMPONENT ...') print(component) print('## EXPECTED ...') pprint(expected_fns) print('## RESULT ...') pprint(cmr_fns) print('## STRATEGIES ..') pprint(CM.strategy) pprint(CM.strategies) print('--------------------------------') res = raw_input('Is the result correct? (y/n/s/d): ') if res.lower() in ['y', 'yes']: MATCH_MAP[component] = cmr_fns EXPECTED[iw.html_url] = cmr_fns save_expected(EXPECTED) continue elif res.lower() in ['s', 'skip']: SKIP.append(hurl) save_skip(SKIP) continue elif res.lower() in ['d', 'debug']: import epdb; epdb.st() ERRORS.append(iw.html_url) ERRORS_COMPONENTS.append( { 'url': iw.html_url, 'component': component, 'component_raw': iw.template_data.get('component_raw'), 'result': cmr_fns, 'expected': expected_fns, 'strategy': CM.strategy, 'strategies': CM.strategies } ) else: if component not in MATCH_MAP: MATCH_MAP[component] = cmr_fns save_match_map(MATCH_MAP) if hurl not in EXPECTED: EXPECTED[hurl] = cmr_fns save_expected(EXPECTED) continue pprint(ERRORS) fn = os.path.join(FIXTUREDIR, 'component_errors.json') with open(fn, 'wb') as f: f.write(json.dumps(ERRORS_COMPONENTS, indent=2, sort_keys=True)) clean_metafiles(METAFILES)
def main(): tocheck = [ #32226, #30361, #31006, #58674, #63611, #64320, #66891, #68784, 69010, ] redirect = set() noredirect = set() nometa = set() cachedir = '/home/jtanner/.ansibullbot/cache' gitrepo = GitRepoWrapper(cachedir=cachedir, repo='https://github.com/ansible/ansible', commit=None, rebase=False) rdata = gitrepo.get_file_content(u'.github/BOTMETA.yml') botmeta = BotMetadataParser.parse_yaml(rdata) cm = AnsibleComponentMatcher(cachedir=cachedir, gitrepo=gitrepo, botmeta=botmeta, botmetafile=None, email_cache=None, usecache=True, use_galaxy=True) ''' mr = parse_match_results() for issue in sorted(mr.keys(), key=lambda x: int(x.split('/')[-1]), reverse=True): print(issue) number = int(issue.split('/')[-1]) #if number != 68709: # continue print(number) mfile = os.path.join('~/.ansibullbot/cache/ansible/ansible/issues/%s' % number, 'meta.json') mfile = os.path.expanduser(mfile) if os.path.exists(mfile): with open(mfile, 'r') as f: imeta = json.loads(f.read()) else: nometa.add(issue) imeta = {} if imeta: iw = MockIssueWrapper(issue, meta=imeta) cfacts = get_collection_facts(iw, cm, imeta) #pprint(cfacts) if cfacts.get('needs_collection_redirect') == True: redirect.add(issue) else: noredirect.add(issue) #if not imeta['is_backport']: # import epdb; epdb.st() ''' mmap = {} #gmatches = cm.search_ecosystem('contrib/inventory/ec2.py') #import epdb; epdb.st() mfiles = get_issues() for mfile in mfiles: with open(mfile, 'r') as f: imeta = json.loads(f.read()) print(imeta['html_url']) number = int(imeta['html_url'].split('/')[-1]) if number not in tocheck: continue newmeta = copy.deepcopy(imeta) iw = MockIssueWrapper(imeta['html_url'], meta=newmeta, gitrepo=gitrepo) #cmatches = cm.match_components(iw.title, iw.body, iw.component) cmmeta = get_component_match_facts(iw, cm, []) newmeta.update(cmmeta) cfmeta = get_collection_facts(iw, cm, newmeta) # check api deltas ... #cm1 = cm.match(iw) #cm2 = cm.match_components(iw.title, iw.body, iw.component, files=iw.files) #import epdb; epdb.st() print('component: %s' % iw.component) print(cmmeta['component_filenames']) #pprint(cfmeta) cf2vals = [x for x in list(cfmeta['collection_filemap'].values()) if x] cf1vals = [x for x in list(imeta['collection_filemap'].values()) if x] ''' if cf1vals or cf2vals: pprint(cf1vals) pprint(cf2vals) #import epdb; epdb.st() ''' ''' if cf2vals != cf1vals: pprint(cf1vals) pprint(cf2vals) import epdb; epdb.st() ''' pprint(cfmeta) import epdb epdb.st() print('# %s total issues|PRs without meta' % len(list(nometa))) print('# %s total issues|PRs not redirected to collections' % len(list(noredirect))) print('# %s total issues|PRs redirected to collections' % len(list(redirect))) import epdb epdb.st()