def runTest(self): fn = 'metadata_1.yml' fn = os.path.join(os.path.dirname(__file__), fn) with open(fn, 'rb') as f: data = f.read() pdata = BotMetadataParser.parse_yaml(data)
def test_keywords(self): LABELS_SUPPORT_PROPAGATION = """ macros: module_utils: lib/ansible/module_utils files: $module_utils/network/: support: network labels: networking $module_utils/network/fw: labels: firewall $module_utils/network/fw/sub: support: core labels: [fwsub] $module_utils/network/fw/sub/childA: $module_utils/network/fw/sub/childB: support: another_level labels: labelB $module_utils/network/iwfu.py: support: community labels: firewall $module_utils/other: labels: thing """ data = BotMetadataParser.parse_yaml(LABELS_SUPPORT_PROPAGATION) self._test(data)
def parse_metadata(self): if self.botmetafile is not None: with open(self.botmetafile, 'rb') as f: rdata = f.read() else: fp = '.github/BOTMETA.yml' rdata = self.get_file_content(fp) if rdata: self.botmeta = BotMetadataParser.parse_yaml(rdata) else: self.botmeta = {} # reshape meta into old format self.CMAP = {} for k, v in self.botmeta.get('files', {}).items(): if not v: continue if 'keywords' not in v: continue for keyword in v['keywords']: if keyword not in self.CMAP: self.CMAP[keyword] = [] if k not in self.CMAP[keyword]: self.CMAP[keyword].append(k) # update the data self.get_files() self.get_filemap()
def runTest(self): data = BotMetadataParser.parse_yaml(EXAMPLE1) assert 'macros' in data assert 'files' in data assert 'lib/ansible/cli/galaxy/' in data['files'] assert 'lib/ansible/cli/vault.py' in data['files'] assert 'lib/ansible/parsing/vault/' in data['files'] assert 'lib/ansible/foobar/' in data['files'] self.assertEqual(data['files']['lib/ansible/foobar/']['labels'], ['ansible', 'bar', 'foo', 'foobar', 'lib']) self.assertEqual( data['files']['lib/ansible/cli/vault.py']['maintainers'], ['larry', 'curly', 'moe', 'jeff']) # double-macro assert 'lib/ansible/modules/x/y' in data['files'] assert 'maintainers' in data['files']['lib/ansible/modules/x/y'] self.assertEqual( data['files']['lib/ansible/modules/x/y']['maintainers'], ['steven']) assert 'team_oneline' in data['macros'] assert isinstance(data['macros']['team_oneline'], list) self.assertEqual(data['macros']['team_oneline'], ['one', 'line', 'at', 'a', 'time'])
def parse_metadata(self): if self.botmetafile is not None: with open(self.botmetafile, 'rb') as f: rdata = f.read() else: fp = u'.github/BOTMETA.yml' rdata = self.get_file_content(fp) if rdata: self.botmeta = BotMetadataParser.parse_yaml(rdata) else: self.botmeta = {} # reshape meta into old format self.CMAP = {} for k, v in self.botmeta.get(u'files', {}).items(): if not v: continue if u'keywords' not in v: continue for keyword in v[u'keywords']: if keyword not in self.CMAP: self.CMAP[keyword] = [] if k not in self.CMAP[keyword]: self.CMAP[keyword].append(k) # update the data self.get_files() self.get_filemap()
def runTest(self): fn = 'metadata_1.yml' fn = os.path.join(os.path.dirname(__file__), fn) with open(fn, 'rb') as f: data = f.read() pdata = BotMetadataParser.parse_yaml(data)
def test_keywords_order(self): """Check that: - labels are inherited - support keyword is inherited when not set Use ruamel in order to check that order does't count """ LABELS_SUPPORT_PROPAGATION = """ macros: module_utils: lib/ansible/module_utils files: $module_utils/network/fw/sub/childA: $module_utils/network/fw/sub/childB: support: another_level labels: labelB $module_utils/network/fw/sub: support: core labels: [fwsub] $module_utils/other: labels: thing $module_utils/network/iwfu.py: support: community labels: firewall $module_utils/network/: support: network labels: networking $module_utils/network/fw: labels: firewall """ data = BotMetadataParser.parse_yaml(LABELS_SUPPORT_PROPAGATION) self.assertIsInstance(data, ruamel.yaml.comments.CommentedMap) # ensure mock is effective self._test(data)
def runTest(self): data = BotMetadataParser.parse_yaml(EXAMPLE1) assert u'macros' in data assert u'files' in data assert u'lib/ansible/cli/galaxy' in data[u'files'] assert u'lib/ansible/cli/vault.py' in data[u'files'] assert u'lib/ansible/parsing/vault' in data[u'files'] assert u'lib/ansible/foobar' in data[u'files'] self.assertEqual(data[u'files'][u'lib/ansible/foobar'][u'labels'], [u'ansible', u'bar', u'foo', u'foobar', u'lib']) self.assertEqual( data[u'files'][u'lib/ansible/cli/vault.py'][u'maintainers'], [u'larry', u'curly', u'moe', u'jeff'], ) # double-macro assert u'lib/ansible/modules/x/y' in data[u'files'] assert u'maintainers' in data[u'files'][u'lib/ansible/modules/x/y'] self.assertEqual( data[u'files'][u'lib/ansible/modules/x/y'][u'maintainers'], [u'steven']) assert u'team_oneline' in data[u'macros'] assert isinstance(data[u'macros'][u'team_oneline'], list) self.assertEqual(data[u'macros'][u'team_oneline'], [u'one', u'line', u'at', u'a', u'time']) self.assertEqual(dict, type(data[u'files'][u'packaging']))
def load_meta(self): if self.botmetafile is not None: with open(self.botmetafile, 'rb') as f: rdata = f.read() else: fp = '.github/BOTMETA.yml' rdata = self.gitrepo.get_file_content(fp) self.BOTMETA = BotMetadataParser.parse_yaml(rdata)
def parse_metadata(self): fp = '.github/BOTMETA.yml' rdata = self.get_file_content(fp) self.botmeta = BotMetadataParser.parse_yaml(rdata) # load the modules logging.info('loading modules') self.get_ansible_modules()
def parse_metadata(self): if self.botmetafile is not None: with open(self.botmetafile, 'rb') as f: rdata = f.read() else: fp = u'.github/BOTMETA.yml' rdata = self.get_file_content(fp) self.botmeta = BotMetadataParser.parse_yaml(rdata) # load the modules logging.info(u'loading modules') self.get_ansible_modules()
def parse_metadata(self): if self.botmetafile is not None: with open(self.botmetafile, 'rb') as f: rdata = f.read() else: fp = u'.github/BOTMETA.yml' rdata = self.get_file_content(fp) self.botmeta = BotMetadataParser.parse_yaml(rdata) # load the modules logging.info(u'loading modules') self.get_ansible_modules()
def parse_metadata(self): fp = '.github/BOTMETA.yml' rdata = self.get_file_content(fp) self.botmeta = BotMetadataParser.parse_yaml(rdata) # reshape data to the old format self.maintainers = {} for k, v in self.botmeta['files'].items(): fp = k.replace('lib/ansible/modules/', '') if isinstance(v, dict): self.maintainers[fp] = v.get('maintainers', []) else: self.maintainers[fp] = [] # load the modules logging.info('loading modules') self.get_ansible_modules()
def runTest(self): data = BotMetadataParser.parse_yaml(EXAMPLE_ANCHORS) # shortcuts topdir = u'lib/ansible/modules/topdir' dfile = u'docs/foo/bar' mfile = u'lib/ansible/modules/topdir/topfile' # labels should be automatic from the path(s) assert u'docs' in data[u'files'][dfile][u'labels'] # children should inherit from their parent anchors assert u'topdir' in data[u'files'][dfile][u'labels'] assert u'topdir' in data[u'files'][dfile][u'labels'] # we do not want pointers merging all data into the anchor assert u'docs' not in data[u'files'][topdir][u'labels'] assert u'docs' not in data[u'files'][mfile][u'labels']
def runTest(self): data = BotMetadataParser.parse_yaml(EXAMPLE_ANCHORS) # shortcuts topdir = 'lib/ansible/modules/topdir' dfile = 'docs/foo/bar' mfile = 'lib/ansible/modules/topdir/topfile' # labels should be automatic from the path(s) assert 'docs' in data['files'][dfile]['labels'] # children should inherit from their parent anchors assert 'topdir' in data['files'][dfile]['labels'] assert 'topdir' in data['files'][dfile]['labels'] # we do not want pointers merging all data into the anchor assert 'docs' not in data['files'][topdir]['labels'] assert 'docs' not in data['files'][mfile]['labels']
def parse_metadata(self): fp = '.github/BOTMETA.yml' rdata = self.get_file_content(fp) self.botmeta = BotMetadataParser.parse_yaml(rdata) # reshape meta into old format self.CMAP = {} for k,v in self.botmeta['files'].items(): if not v: continue if 'keywords' not in v: continue for keyword in v['keywords']: if keyword not in self.CMAP: self.CMAP[keyword] = [] if k not in self.CMAP[keyword]: self.CMAP[keyword].append(k) # update the data self.get_files() self.get_filemap()
def runTest(self): data = BotMetadataParser.parse_yaml(EXAMPLE1) assert u'macros' in data assert u'files' in data assert u'lib/ansible/cli/galaxy' in data[u'files'] assert u'lib/ansible/cli/vault.py' in data[u'files'] assert u'lib/ansible/parsing/vault' in data[u'files'] assert u'lib/ansible/foobar' in data[u'files'] self.assertEqual( data[u'files'][u'lib/ansible/foobar'][u'labels'], [u'ansible', u'bar', u'foo', u'foobar', u'lib'] ) self.assertEqual( data[u'files'][u'lib/ansible/cli/vault.py'][u'maintainers'], [u'larry', u'curly', u'moe', u'jeff'], ) # double-macro assert u'lib/ansible/modules/x/y' in data[u'files'] assert u'maintainers' in data[u'files'][u'lib/ansible/modules/x/y'] self.assertEqual( data[u'files'][u'lib/ansible/modules/x/y'][u'maintainers'], [u'steven'] ) assert u'team_oneline' in data[u'macros'] assert isinstance(data[u'macros'][u'team_oneline'], list) self.assertEqual( data[u'macros'][u'team_oneline'], [u'one', u'line', u'at', u'a', u'time'] ) self.assertEqual(dict, type(data[u'files'][u'packaging']))
def test_supported_by_order(self): """Check that: - supported_by keyword is inherited when not set Use ruamel in order to check that order does't count """ LABELS_SUPPORTED_BY_PROPAGATION = """ macros: module_utils: lib/ansible/module_utils files: $module_utils/network/fw/sub/childA: $module_utils/network/fw/sub/childB: supported_by: another_level labels: labelB $module_utils/network/fw/sub: supported_by: core labels: [fwsub] $module_utils/other: labels: thing $module_utils/network/iwfu.py: supported_by: community labels: firewall $module_utils/network/: supported_by: network labels: networking $module_utils/network/fw: labels: firewall """ data = BotMetadataParser.parse_yaml(LABELS_SUPPORTED_BY_PROPAGATION) self.assertIsInstance(data, ruamel.yaml.comments.CommentedMap) # ensure mock is effective #### 'supported_by' key self.assertEqual( data[u'files'][u'lib/ansible/module_utils/network'][u'supported_by'], [u'network'] ) # subpath: supported_by key is inherited self.assertEqual( data[u'files'][u'lib/ansible/module_utils/network/fw'][u'supported_by'], [u'network'] ) # subpath: supported_by key is overridden self.assertEqual( data[u'files'][u'lib/ansible/module_utils/network/iwfu.py'][u'supported_by'], [u'community'] ) # subpath: supported_by key is overridden self.assertEqual( data[u'files'][u'lib/ansible/module_utils/network/fw/sub'][u'supported_by'], [u'core'] ) # subpath: supported_by key is inherited self.assertEqual( data[u'files'][u'lib/ansible/module_utils/network/fw/sub/childA'][u'supported_by'], [u'core'] ) # subpath: supported_by key is overridden self.assertEqual( data[u'files'][u'lib/ansible/module_utils/network/fw/sub/childB'][u'supported_by'], [u'another_level'] ) # default value for supported_by isn't set by botmeta self.assertNotIn(u'supported_by', data[u'files'][u'lib/ansible/module_utils/other'])
def test_keywords(self): LABELS_SUPPORT_PROPAGATION = """ macros: module_utils: lib/ansible/module_utils files: $module_utils/network/: support: network labels: networking $module_utils/network/fw: labels: firewall $module_utils/network/fw/sub: support: core labels: [fwsub] $module_utils/network/fw/sub/childA: $module_utils/network/fw/sub/childB: support: another_level labels: labelB $module_utils/network/iwfu.py: support: community labels: firewall $module_utils/other: labels: thing """ data = BotMetadataParser.parse_yaml(LABELS_SUPPORT_PROPAGATION) assert u'macros' in data assert u'files' in data self.assertEqual(set([u'lib/ansible/module_utils/network', u'lib/ansible/module_utils/network/fw', u'lib/ansible/module_utils/network/fw/sub', u'lib/ansible/module_utils/network/fw/sub/childA', u'lib/ansible/module_utils/network/fw/sub/childB', u'lib/ansible/module_utils/network/iwfu.py', u'lib/ansible/module_utils/other']), set(data[u'files'].keys())) #### 'labels' key self.assertEqual( set(data[u'files'][u'lib/ansible/module_utils/network'][u'labels']), set([u'lib', u'ansible', u'module_utils', u'network', u'networking']) ) self.assertEqual( set(data[u'files'][u'lib/ansible/module_utils/network/fw'][u'labels']), set([u'lib', u'ansible', u'module_utils', u'network', u'networking', u'fw', u'firewall']) ) self.assertEqual( set(data[u'files'][u'lib/ansible/module_utils/network/fw/sub'][u'labels']), set([u'lib', u'ansible', u'module_utils', u'network', u'networking', u'fw', u'firewall', u'sub', u'fwsub']) ) self.assertEqual( set(data[u'files'][u'lib/ansible/module_utils/network/fw/sub/childA'][u'labels']), set([u'lib', u'ansible', u'module_utils', u'network', u'networking', u'fw', u'firewall', u'sub', u'fwsub', u'childA']) ) self.assertEqual( set(data[u'files'][u'lib/ansible/module_utils/network/fw/sub/childB'][u'labels']), set([u'lib', u'ansible', u'module_utils', u'network', u'networking', u'fw', u'firewall', u'sub', u'fwsub', u'childB', u'labelB']) ) self.assertEqual( set(data[u'files'][u'lib/ansible/module_utils/network/iwfu.py'][u'labels']), set([u'lib', u'ansible', u'module_utils', u'network', u'networking', u'iwfu', u'firewall']) ) #### 'support' key self.assertEqual( data[u'files'][u'lib/ansible/module_utils/network'][u'support'], [u'network'] ) # subpath: support key is inherited self.assertEqual( data[u'files'][u'lib/ansible/module_utils/network/fw'][u'support'], [u'network'] ) # subpath: support key is overridden self.assertEqual( data[u'files'][u'lib/ansible/module_utils/network/iwfu.py'][u'support'], [u'community'] ) # subpath: support key is overridden self.assertEqual( data[u'files'][u'lib/ansible/module_utils/network/fw/sub'][u'support'], [u'core'] ) # subpath: support key is inherited self.assertEqual( data[u'files'][u'lib/ansible/module_utils/network/fw/sub/childA'][u'support'], [u'core'] ) # subpath: support key is overridden self.assertEqual( data[u'files'][u'lib/ansible/module_utils/network/fw/sub/childB'][u'support'], [u'another_level'] ) # default value for support isn't set by botmeta self.assertNotIn(u'support', data[u'files'][u'lib/ansible/module_utils/other'])
def main(): tocheck = [ #32226, #30361, #31006, #58674, #63611, #64320, #66891, #68784, 69010, ] redirect = set() noredirect = set() nometa = set() cachedir = '/home/jtanner/.ansibullbot/cache' gitrepo = GitRepoWrapper(cachedir=cachedir, repo='https://github.com/ansible/ansible', commit=None, rebase=False) rdata = gitrepo.get_file_content(u'.github/BOTMETA.yml') botmeta = BotMetadataParser.parse_yaml(rdata) cm = AnsibleComponentMatcher(cachedir=cachedir, gitrepo=gitrepo, botmeta=botmeta, botmetafile=None, email_cache=None, usecache=True, use_galaxy=True) ''' mr = parse_match_results() for issue in sorted(mr.keys(), key=lambda x: int(x.split('/')[-1]), reverse=True): print(issue) number = int(issue.split('/')[-1]) #if number != 68709: # continue print(number) mfile = os.path.join('~/.ansibullbot/cache/ansible/ansible/issues/%s' % number, 'meta.json') mfile = os.path.expanduser(mfile) if os.path.exists(mfile): with open(mfile, 'r') as f: imeta = json.loads(f.read()) else: nometa.add(issue) imeta = {} if imeta: iw = MockIssueWrapper(issue, meta=imeta) cfacts = get_collection_facts(iw, cm, imeta) #pprint(cfacts) if cfacts.get('needs_collection_redirect') == True: redirect.add(issue) else: noredirect.add(issue) #if not imeta['is_backport']: # import epdb; epdb.st() ''' mmap = {} #gmatches = cm.search_ecosystem('contrib/inventory/ec2.py') #import epdb; epdb.st() mfiles = get_issues() for mfile in mfiles: with open(mfile, 'r') as f: imeta = json.loads(f.read()) print(imeta['html_url']) number = int(imeta['html_url'].split('/')[-1]) if number not in tocheck: continue newmeta = copy.deepcopy(imeta) iw = MockIssueWrapper(imeta['html_url'], meta=newmeta, gitrepo=gitrepo) #cmatches = cm.match_components(iw.title, iw.body, iw.component) cmmeta = get_component_match_facts(iw, cm, []) newmeta.update(cmmeta) cfmeta = get_collection_facts(iw, cm, newmeta) # check api deltas ... #cm1 = cm.match(iw) #cm2 = cm.match_components(iw.title, iw.body, iw.component, files=iw.files) #import epdb; epdb.st() print('component: %s' % iw.component) print(cmmeta['component_filenames']) #pprint(cfmeta) cf2vals = [x for x in list(cfmeta['collection_filemap'].values()) if x] cf1vals = [x for x in list(imeta['collection_filemap'].values()) if x] ''' if cf1vals or cf2vals: pprint(cf1vals) pprint(cf2vals) #import epdb; epdb.st() ''' ''' if cf2vals != cf1vals: pprint(cf1vals) pprint(cf2vals) import epdb; epdb.st() ''' pprint(cfmeta) import epdb epdb.st() print('# %s total issues|PRs without meta' % len(list(nometa))) print('# %s total issues|PRs not redirected to collections' % len(list(noredirect))) print('# %s total issues|PRs redirected to collections' % len(list(redirect))) import epdb epdb.st()
def test_keywords(self): LABELS_SUPPORT_PROPAGATION = """ macros: module_utils: lib/ansible/module_utils files: $module_utils/network/: support: network labels: networking $module_utils/network/fw: labels: firewall $module_utils/network/fw/sub: support: core labels: [fwsub] $module_utils/network/fw/sub/childA: $module_utils/network/fw/sub/childB: support: another_level labels: labelB $module_utils/network/iwfu.py: support: community labels: firewall $module_utils/other: labels: thing """ data = BotMetadataParser.parse_yaml(LABELS_SUPPORT_PROPAGATION) assert 'macros' in data assert 'files' in data self.assertEqual( { 'lib/ansible/module_utils/network', 'lib/ansible/module_utils/network/fw', 'lib/ansible/module_utils/network/fw/sub', 'lib/ansible/module_utils/network/fw/sub/childA', 'lib/ansible/module_utils/network/fw/sub/childB', 'lib/ansible/module_utils/network/iwfu.py', 'lib/ansible/module_utils/other' }, set(data['files'].keys())) #### 'labels' key self.assertEqual( set(data['files']['lib/ansible/module_utils/network']['labels']), {'lib', 'ansible', 'module_utils', 'network', 'networking'}) self.assertEqual( set(data['files']['lib/ansible/module_utils/network/fw'] ['labels']), { 'lib', 'ansible', 'module_utils', 'network', 'networking', 'fw', 'firewall' }) self.assertEqual( set(data['files']['lib/ansible/module_utils/network/fw/sub'] ['labels']), { 'lib', 'ansible', 'module_utils', 'network', 'networking', 'fw', 'firewall', 'sub', 'fwsub' }) self.assertEqual( set(data['files']['lib/ansible/module_utils/network/fw/sub/childA'] ['labels']), { 'lib', 'ansible', 'module_utils', 'network', 'networking', 'fw', 'firewall', 'sub', 'fwsub', 'childA' }) self.assertEqual( set(data['files']['lib/ansible/module_utils/network/fw/sub/childB'] ['labels']), { 'lib', 'ansible', 'module_utils', 'network', 'networking', 'fw', 'firewall', 'sub', 'fwsub', 'childB', 'labelB' }) self.assertEqual( set(data['files']['lib/ansible/module_utils/network/iwfu.py'] ['labels']), { 'lib', 'ansible', 'module_utils', 'network', 'networking', 'iwfu', 'firewall' }) #### 'support' key self.assertEqual( data['files']['lib/ansible/module_utils/network']['support'], ['network']) # subpath: support key is inherited self.assertEqual( data['files']['lib/ansible/module_utils/network/fw']['support'], ['network']) # subpath: support key is overridden self.assertEqual( data['files']['lib/ansible/module_utils/network/iwfu.py'] ['support'], ['community']) # subpath: support key is overridden self.assertEqual( data['files']['lib/ansible/module_utils/network/fw/sub'] ['support'], ['core']) # subpath: support key is inherited self.assertEqual( data['files']['lib/ansible/module_utils/network/fw/sub/childA'] ['support'], ['core']) # subpath: support key is overridden self.assertEqual( data['files']['lib/ansible/module_utils/network/fw/sub/childB'] ['support'], ['another_level']) # default value for support isn't set by botmeta self.assertNotIn('support', data['files']['lib/ansible/module_utils/other'])