def get_shipit_facts(issuewrapper,
meta,
module_indexer,
core_team=[],
botnames=[]):
""" Count shipits by maintainers/community/other """
# maintainers - people who maintain this file/module
# community - people who maintain file(s) in the same directory
# other - anyone else who comments with shipit/+1/LGTM
iw = issuewrapper
nmeta = {
'shipit': False,
'owner_pr': False,
'shipit_ansible': False,
'shipit_community': False,
'shipit_count_other': False,
'shipit_count_community': False,
'shipit_count_maintainer': False,
'shipit_count_ansible': False,
'shipit_actors': None,
'community_usernames': [],
'notify_community_shipit': False,
}
if not iw.is_pullrequest():
return nmeta
module_utils_files_owned = 0 # module_utils files for which submitter is maintainer
if meta['is_module_util']:
for f in iw.files:
if f.startswith('lib/ansible/module_utils'
) and f in module_indexer.botmeta['files']:
maintainers = module_indexer.botmeta['files'][f].get(
'maintainers', [])
if maintainers and (iw.submitter in maintainers):
module_utils_files_owned += 1
if module_utils_files_owned == len(iw.files):
nmeta['owner_pr'] = True
return nmeta
if not meta['module_match']:
return nmeta
# https://github.com/ansible/ansibullbot/issues/722
if iw.wip:
logging.debug('WIP PRs do not get shipits')
return nmeta
if meta['is_needs_revision'] or meta['is_needs_rebase']:
logging.debug(
'PRs with needs_revision or needs_rebase label do not get shipits')
return nmeta
maintainers = meta['module_match']['maintainers']
maintainers = \
ModuleIndexer.replace_ansible(
maintainers,
core_team,
bots=botnames
)
modules_files_owned = 0
if not meta['is_new_module']:
for f in iw.files:
if f.startswith(
'lib/ansible/modules'
) and iw.submitter in module_indexer.modules[f]['maintainers']:
modules_files_owned += 1
nmeta['owner_pr'] = modules_files_owned + module_utils_files_owned == len(
iw.files)
# community is the other maintainers in the same namespace
mnamespace = meta['module_match']['namespace']
community = \
module_indexer.get_maintainers_for_namespace(mnamespace)
community = [
x for x in community
if x != 'ansible' and x not in core_team and x != 'DEPRECATED'
]
# shipit tallies
ansible_shipits = 0
maintainer_shipits = 0
community_shipits = 0
other_shipits = 0
shipit_actors = []
shipit_actors_other = []
for event in iw.history.history:
if event['event'] not in [
'commented', 'committed', 'review_approved', 'review_comment'
]:
continue
if event['actor'] in botnames:
continue
# commits reset the counters
if event['event'] == 'committed':
ansible_shipits = 0
maintainer_shipits = 0
community_shipits = 0
other_shipits = 0
shipit_actors = []
shipit_actors_other = []
continue
actor = event['actor']
body = event.get('body', '')
body = body.strip()
if not is_approval(body):
continue
logging.info('%s shipit' % actor)
# ansible shipits
if actor in core_team:
if actor not in shipit_actors:
ansible_shipits += 1
shipit_actors.append(actor)
continue
# maintainer shipits
if actor in maintainers:
if actor not in shipit_actors:
maintainer_shipits += 1
shipit_actors.append(actor)
continue
# community shipits
if actor in community:
if actor not in shipit_actors:
community_shipits += 1
shipit_actors.append(actor)
continue
# other shipits
if actor not in shipit_actors_other:
other_shipits += 1
shipit_actors_other.append(actor)
continue
# submitters should count if they are core team/maintainers/community
if iw.submitter in core_team:
if iw.submitter not in shipit_actors:
ansible_shipits += 1
shipit_actors.append(iw.submitter)
elif iw.submitter in maintainers:
if iw.submitter not in shipit_actors:
maintainer_shipits += 1
shipit_actors.append(iw.submitter)
elif iw.submitter in community:
if iw.submitter not in shipit_actors:
community_shipits += 1
shipit_actors.append(iw.submitter)
nmeta['shipit_count_other'] = other_shipits
nmeta['shipit_count_community'] = community_shipits
nmeta['shipit_count_maintainer'] = maintainer_shipits
nmeta['shipit_count_ansible'] = ansible_shipits
nmeta['shipit_actors'] = shipit_actors
nmeta['shipit_actors_other'] = shipit_actors_other
nmeta['community_usernames'] = sorted(community)
total = community_shipits + maintainer_shipits + ansible_shipits
# include shipits from other people to push over the edge
if total == 1 and other_shipits > 2:
total += other_shipits
if total > 1:
nmeta['shipit'] = True
elif meta['is_new_module'] or \
(len(maintainers) == 1 and maintainer_shipits == 1):
if community:
bpc = iw.history.get_boilerplate_comments()
if 'community_shipit_notify' not in bpc:
nmeta['notify_community_shipit'] = True
logging.info('total shipits: %s' % total)
return nmeta
def get_shipit_facts(issuewrapper,
inmeta,
module_indexer,
core_team=[],
botnames=[]):
""" Count shipits by maintainers/community/other """
# supershipit - maintainers with isolated commit access
# maintainers - people who maintain this file/module
# community - people who maintain file(s) in the same directory
# other - anyone else who comments with shipit/+1/LGTM
meta = inmeta.copy()
iw = issuewrapper
nmeta = {
u'shipit': False,
u'supershipit': False,
u'owner_pr': False,
u'shipit_ansible': False,
u'shipit_community': False,
u'shipit_count_other': False,
u'shipit_count_community': False,
u'shipit_count_maintainer': False,
u'shipit_count_ansible': False,
u'shipit_count_vtotal': False,
u'shipit_count_historical': False,
u'shipit_actors': None,
u'supershipit_actors': None,
u'community_usernames': [],
u'notify_community_shipit': False,
u'is_rebuild_merge': False,
}
if not iw.is_pullrequest():
return nmeta
# https://github.com/ansible/ansibullbot/issues/1147
meta[u'component_matches'] = [
x for x in meta.get(u'component_matches', [])
if not x[u'repo_filename'].startswith(u'changelogs/fragments/')
]
files = [f for f in iw.files if not f.startswith(u'changelogs/fragments/')]
module_utils_files_owned = 0 # module_utils files for which submitter is maintainer
if meta[u'is_module_util']:
for f in files:
if f.startswith(u'lib/ansible/module_utils'
) and f in module_indexer.botmeta[u'files']:
maintainers = module_indexer.botmeta[u'files'][f].get(
u'maintainers', [])
if maintainers and (iw.submitter in maintainers):
module_utils_files_owned += 1
modules_files_owned = 0
if not meta[u'is_new_module']:
for f in files:
if f.startswith(u'lib/ansible/modules') and iw.submitter in meta[
u'component_maintainers']:
modules_files_owned += 1
nmeta[u'owner_pr'] = modules_files_owned + module_utils_files_owned == len(
files)
# https://github.com/ansible/ansibullbot/issues/722
if iw.wip:
logging.debug(u'WIP PRs do not get shipits')
return nmeta
if meta[u'is_needs_revision'] or meta[u'is_needs_rebase']:
logging.debug(
u'PRs with needs_revision or needs_rebase label do not get shipits'
)
return nmeta
supershipiteers_byfile = {}
supershipiteers_byuser = {}
for cm in meta.get('component_matches', []):
_ss = cm.get(u'supershipit', [])
supershipiteers_byfile[cm[u'repo_filename']] = _ss[:]
for ss in _ss:
if ss not in supershipiteers_byuser:
supershipiteers_byuser[ss] = []
supershipiteers_byuser[ss].append(cm[u'repo_filename'])
maintainers = meta.get(u'component_maintainers', [])
maintainers = \
ModuleIndexer.replace_ansible(
maintainers,
core_team,
bots=botnames
)
# community is the other maintainers in the same namespace
community = meta.get(u'component_namespace_maintainers', [])
community = [
x for x in community
if x != u'ansible' and x not in core_team and x != u'DEPRECATED'
]
# shipit tallies
ansible_shipits = 0
maintainer_shipits = 0
community_shipits = 0
other_shipits = 0
shipit_actors = []
shipit_actors_other = []
supershipiteers_voted = set()
rebuild_merge = False
shipits_historical = set()
for event in iw.history.history:
if event[u'event'] not in [
u'commented', u'committed', u'review_approved',
u'review_comment'
]:
continue
if event[u'actor'] in botnames:
continue
# commits reset the counters
if event[u'event'] == u'committed':
logging.info(event)
ansible_shipits = 0
maintainer_shipits = 0
community_shipits = 0
other_shipits = 0
shipit_actors = []
shipit_actors_other = []
supershipiteers_voted = set()
rebuild_merge = False
logging.info('commit detected, resetting shipit tallies')
continue
actor = event[u'actor']
body = event.get(u'body', u'')
body = body.strip()
if not is_approval(body):
continue
# historical shipits (keep track of all of them, even if reset)
shipits_historical.add(actor)
if actor in core_team and is_rebuild_merge(body):
rebuild_merge = True
logging.info(u'%s shipit [rebuild_merge]' % actor)
else:
logging.info(u'%s shipit' % actor)
# super shipits
if actor in supershipiteers_byuser:
supershipiteers_voted.add(actor)
# ansible shipits
if actor in core_team:
if actor not in shipit_actors:
ansible_shipits += 1
shipit_actors.append(actor)
continue
# maintainer shipits
if actor in maintainers:
if actor not in shipit_actors:
maintainer_shipits += 1
shipit_actors.append(actor)
continue
# community shipits
if actor in community:
if actor not in shipit_actors:
community_shipits += 1
shipit_actors.append(actor)
continue
# other shipits
if actor not in shipit_actors_other:
other_shipits += 1
shipit_actors_other.append(actor)
continue
# submitters should count if they are core team/maintainers/community
if iw.submitter in core_team:
if iw.submitter not in shipit_actors:
ansible_shipits += 1
shipit_actors.append(iw.submitter)
shipits_historical.add(iw.submitter)
elif iw.submitter in maintainers:
if iw.submitter not in shipit_actors:
maintainer_shipits += 1
shipit_actors.append(iw.submitter)
shipits_historical.add(iw.submitter)
elif iw.submitter in community:
if iw.submitter not in shipit_actors:
community_shipits += 1
shipit_actors.append(iw.submitter)
shipits_historical.add(iw.submitter)
nmeta[u'shipit_count_other'] = other_shipits
nmeta[u'shipit_count_community'] = community_shipits
nmeta[u'shipit_count_maintainer'] = maintainer_shipits
nmeta[u'shipit_count_ansible'] = ansible_shipits
nmeta[u'shipit_actors'] = shipit_actors
nmeta[u'shipit_actors_other'] = shipit_actors_other
nmeta[u'community_usernames'] = sorted(community)
nmeta[u'shipit_count_historical'] = list(shipits_historical)
nmeta[u'shipit_count_htotal'] = len(list(shipits_historical))
total = community_shipits + maintainer_shipits + ansible_shipits
nmeta[u'shipit_count_vtotal'] = total + other_shipits
if rebuild_merge:
nmeta['is_rebuild_merge'] = True
# include shipits from other people to push over the edge
if total == 1 and other_shipits > 2:
total += other_shipits
if total > 1 or rebuild_merge:
nmeta[u'shipit'] = True
elif meta[u'is_new_module'] or \
(len(maintainers) == 1 and maintainer_shipits == 1):
# don't notify if there is no maintainer or if submitter is the only namespace maintainer
if set(community) - {iw.submitter}:
bpc = iw.history.get_boilerplate_comments()
bpc = [x[0] for x in bpc]
if u'community_shipit_notify' not in bpc:
nmeta[u'notify_community_shipit'] = True
logging.info(u'total shipits: %s' % total)
# supershipit ...
# if a supershipiteer for each file exists and has blessed the PR
# on the current commit, then override all shipit tallies and get this PR merged
if supershipiteers_voted:
nmeta[u'supershipit_actors'] = list(supershipiteers_voted)
cm_files = [x[u'repo_filename'] for x in meta[u'component_matches']]
ss_files = set()
for ssv in supershipiteers_voted:
for fn in supershipiteers_byuser[ssv]:
ss_files.add(fn)
if sorted(set(cm_files)) == sorted(set(ss_files)):
logging.info(u'supershipit enabled on %s' % iw.html_url)
nmeta[u'supershipit'] = True
nmeta[u'shipit'] = True
else:
for cm_file in sorted(cm_files):
if cm_file not in ss_files:
logging.info('%s is not governed by supershipit' % cm_file)
return nmeta
def main():
set_logger()
METAFILES = extract_metafiles()
SKIP = load_skip()
EXPECTED = load_expected()
MATCH_MAP = load_match_map()
ERRORS = []
ERRORS_COMPONENTS = []
start_at = None
if len(sys.argv) == 2:
start_at = int(sys.argv[1])
FI = FileIndexer(checkoutdir=CACHEDIR)
with open('/tmp/files.json', 'wb') as f:
f.write(json.dumps(FI.files, indent=2))
GQLC = GithubGraphQLClient(C.DEFAULT_GITHUB_TOKEN)
MI = ModuleIndexer(cachedir=CACHEDIR,
gh_client=GQLC,
blames=False,
commits=False)
CM = AnsibleComponentMatcher(cachedir=CACHEDIR)
for k, v in MI.modules.items():
if k in MATCH_MAP:
MATCH_MAP.pop(k, None)
kname = v.get('name')
if kname not in MATCH_MAP:
MATCH_MAP[kname] = v.get('repo_filename')
if kname + ' module' not in MATCH_MAP:
MATCH_MAP[kname + ' module'] = v.get('repo_filename')
if kname + 'module: ' + kname not in MATCH_MAP:
MATCH_MAP['module: ' + kname] = v.get('repo_filename')
if kname + 'module ' + kname not in MATCH_MAP:
MATCH_MAP['module ' + kname] = v.get('repo_filename')
# /modules/remote_management/foreman/katello.py
pname = k.replace('lib/ansible', '')
if pname not in MATCH_MAP:
MATCH_MAP[pname] = v.get('repo_filename')
# ansible/modules/packaging/os/rpm_key.py
pname = k.replace('lib/', '/')
if pname not in MATCH_MAP:
MATCH_MAP[pname] = v.get('repo_filename')
# /ansible/modules/packaging/os/rpm_key.py
pname = k.replace('lib/', '')
if pname not in MATCH_MAP:
MATCH_MAP[pname] = v.get('repo_filename')
# ansible/lib/ansible/modules/monitoring/monit.py
pname = 'ansible/' + k
if pname not in MATCH_MAP:
MATCH_MAP[pname] = v.get('repo_filename')
# network/f5/bigip_gtm_wide_ip
pname = k.replace('lib/ansible/modules/', '')
pname = pname.replace('.py', '')
pname = pname.replace('.ps1', '')
if pname not in MATCH_MAP:
MATCH_MAP[pname] = v.get('repo_filename')
# network/f5/bigip_gtm_wide_ip.py
pname = k.replace('lib/ansible/modules/', '')
if pname not in MATCH_MAP:
MATCH_MAP[pname] = v.get('repo_filename')
# modules/packaging/os/pkgng.py
pname = k.replace('lib/ansible/', '')
if pname not in MATCH_MAP:
MATCH_MAP[pname] = v.get('repo_filename')
save_match_map(MATCH_MAP)
total = len(METAFILES)
for IDMF, MF in enumerate(METAFILES):
if start_at and IDMF < start_at:
continue
with open(MF, 'rb') as f:
meta = json.loads(f.read())
if not meta.get('is_issue'):
continue
component = meta.get('template_data', {}).get('component_raw')
#if component != 'Module `synchronize`':
#if component != 'Module: include_role':
# continue
if component:
print(f'------------------------------------------ {total}|{IDMF}')
print(meta['html_url'])
print(meta['title'])
print(component)
hurl = meta['html_url']
if hurl in SKIP:
continue
# bad template or bad template parsing
if len(component) > 100:
continue
iw = IssueWrapperMock(meta)
if 'module' not in iw.body.lower(
) and 'module' not in iw.title.lower():
continue
expected_fns = []
# OLD METHOD
if hurl not in EXPECTED and component not in MATCH_MAP:
cmf = get_component_match_facts(iw, meta, FI, MI, LABELS)
expected_fns = cmf.get('module_match')
if not isinstance(expected_fns, list):
expected_fns = [expected_fns]
expected_fns = [x['repo_filename'] for x in expected_fns if x]
if 'component_matches' in cmf:
expected_fns = [
x['filename'] for x in cmf['component_matches']
]
expected_fns = sorted(set(expected_fns))
# NEW METHOD
cmr = CM.match_components(iw.title, iw.body,
iw.template_data.get('component_raw'))
cmr_fns = [x['repo_filename'] for x in cmr if x]
cmr_fns = sorted(set(cmr_fns))
# VALIDATE FROM EXPECTED IF KNOWN
if hurl in EXPECTED:
if EXPECTED[hurl] and not isinstance(EXPECTED[hurl], list):
expected_fns = [EXPECTED[hurl]]
elif EXPECTED[hurl]:
expected_fns = EXPECTED[hurl]
else:
expected_fns = []
# USE THE CACHED MAP
if component in MATCH_MAP:
expected_fns = MATCH_MAP[component]
if not isinstance(expected_fns, list):
expected_fns = [expected_fns]
elif component.lower() in MATCH_MAP:
expected_fns = MATCH_MAP[component.lower()]
if not isinstance(expected_fns, list):
expected_fns = [expected_fns]
elif component.startswith(':\n') and component.endswith(' module'):
mapkey = component.lstrip(':\n')
if mapkey in MATCH_MAP:
expected_fns = MATCH_MAP[mapkey]
if not isinstance(expected_fns, list):
expected_fns = [expected_fns]
# OLD CODE USED ACTION PLUGINS INSTEAD OF MODULES
if expected_fns != cmr_fns and hurl not in EXPECTED:
if len(expected_fns) == 1 and len(
cmr_fns) == 1 and 'plugins/action' in expected_fns[0]:
e_bn = os.path.basename(expected_fns[0])
c_bn = os.path.basename(cmr_fns[0])
if e_bn == c_bn:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
# DOCS URLS
if expected_fns != cmr_fns and hurl not in EXPECTED:
if len(cmr_fns) == 1 and 'lib/ansible/modules' in cmr_fns[0]:
c_bn = os.path.basename(cmr_fns[0])
if f'docs.ansible.com/ansible/latest/{c_bn}_module.html' in component:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
elif CM.strategy in ['search_by_regex_urls']:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
# NXOS ISSUES HAVE NXOS_VERSION HEADER
if '- nxos' in component:
if len(cmr_fns) == 1:
if os.path.basename(cmr_fns[0]).replace('.py',
'') in component:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
#import epdb; epdb.st()
# ODDBALL MODULE COMPONENTS
if len(cmr_fns) == 1 and 'lib/ansible/modules' in cmr_fns[0]:
bn = os.path.basename(cmr_fns[0])
bn = bn.replace('.py', '')
bn = bn.replace('.ps1', '')
if (bn in component or bn.lstrip('_')
in component) and 'module' in component.lower():
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
elif component == '- ' + bn:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
elif component == bn + '.py' or component == bn + '.ps1':
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
elif component == '_' + bn + '.py' or component == '_' + bn + '.ps1':
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
elif component == ':\n' + bn or component == ':\n' + bn.lstrip(
'_'):
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
# 'multiple modules', etc ...
if component in CM.KEYWORDS or component.lower() in CM.KEYWORDS:
if component in CM.KEYWORDS and CM.KEYWORDS[
component] is None and not cmr_fns:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
elif component.lower() in CM.KEYWORDS and CM.KEYWORDS[
component.lower()] is None and not cmr_fns:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
elif len(cmr_fns) == 1 and cmr_fns[0] == CM.KEYWORDS.get(
component):
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
elif len(cmr_fns) == 1 and cmr_fns[0] == CM.KEYWORDS.get(
component.lower()):
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
if component.lstrip('-').strip() in CM.KEYWORDS and len(
cmr_fns) == 1:
cname = component.lstrip('-').strip()
if CM.KEYWORDS[cname] == cmr_fns[0]:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
if component.endswith(' lookup') and len(
cmr_fns
) == 1 and 'lib/ansible/plugins/lookup' in cmr_fns[0]:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
if component.endswith(' inventory script') and len(
cmr_fns) == 1 and 'contrib/inventory' in cmr_fns[0]:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
if component.startswith('ansible/lib') and len(cmr_fns) == 1:
fn = cmr_fns[0]
if 'ansible/' + fn == component:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
if component.endswith(' inventory plugin') and len(cmr_fns) == 1:
fn = cmr_fns[0]
if fn.startswith('lib/ansible/plugins/inventory'):
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
if component == 'ec2.py' and cmr_fns and 'contrib/inventory/ec2.py' in cmr_fns:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
if len(expected_fns) == 1 and len(cmr_fns) == 1:
if os.path.basename(expected_fns[0]) == os.path.basename(
cmr_fns[0]):
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
continue
# COMPARE AND RECORD
if expected_fns != cmr_fns and hurl not in EXPECTED:
if component in MATCH_MAP or component.lower() in MATCH_MAP:
if component.lower() in MATCH_MAP:
mmc = MATCH_MAP[component.lower()]
else:
mmc = MATCH_MAP[component]
if not isinstance(mmc, list):
mmc == [mmc]
if mmc == cmr_fns:
EXPECTED[iw.html_url] = cmr_fns
save_expected(EXPECTED)
continue
print('## COMPONENT ...')
print(component)
print('## EXPECTED ...')
pprint(expected_fns)
print('## RESULT ...')
pprint(cmr_fns)
print('## STRATEGIES ..')
pprint(CM.strategy)
pprint(CM.strategies)
print('--------------------------------')
res = raw_input('Is the result correct? (y/n/s/d): ')
if res.lower() in ['y', 'yes']:
MATCH_MAP[component] = cmr_fns
EXPECTED[iw.html_url] = cmr_fns
save_expected(EXPECTED)
continue
elif res.lower() in ['s', 'skip']:
SKIP.append(hurl)
save_skip(SKIP)
continue
elif res.lower() in ['d', 'debug']:
import epdb
epdb.st()
ERRORS.append(iw.html_url)
ERRORS_COMPONENTS.append({
'url':
iw.html_url,
'component':
component,
'component_raw':
iw.template_data.get('component_raw'),
'result':
cmr_fns,
'expected':
expected_fns,
'strategy':
CM.strategy,
'strategies':
CM.strategies
})
else:
if component not in MATCH_MAP:
MATCH_MAP[component] = cmr_fns
save_match_map(MATCH_MAP)
if hurl not in EXPECTED:
EXPECTED[hurl] = cmr_fns
save_expected(EXPECTED)
continue
pprint(ERRORS)
fn = os.path.join(FIXTUREDIR, 'component_errors.json')
with open(fn, 'wb') as f:
f.write(json.dumps(ERRORS_COMPONENTS, indent=2, sort_keys=True))
clean_metafiles(METAFILES)
def get_shipit_facts(issuewrapper,
meta,
module_indexer,
core_team=[],
botnames=[]):
# shipit/+1/LGTM in comment.body from maintainer
# AUTOMERGE
# * New module, existing namespace: require a "shipit" from some
# other maintainer in the namespace. (Ideally, identify a maintainer
# for the entire namespace.)
# * New module, new namespace: require discussion with the creator
# of the namespace, which will likely be a vendor.
# * And all new modules, of course, go in as "preview" mode.
iw = issuewrapper
nmeta = {
'shipit': False,
'owner_pr': False,
'shipit_ansible': False,
'shipit_community': False,
'shipit_count_community': False,
'shipit_count_maintainer': False,
'shipit_count_ansible': False,
'shipit_actors': None,
'community_usernames': [],
'notify_community_shipit': False,
}
if not iw.is_pullrequest():
return nmeta
if not meta['module_match']:
return nmeta
maintainers = meta['module_match']['maintainers']
maintainers = \
ModuleIndexer.replace_ansible(
maintainers,
core_team,
bots=botnames
)
if not meta['is_new_module'] and iw.submitter in maintainers:
nmeta['owner_pr'] = True
# community is the other maintainers in the same namespace
mnamespace = meta['module_match']['namespace']
community = \
module_indexer.get_maintainers_for_namespace(mnamespace)
community = [
x for x in community
if x != 'ansible' and x not in core_team and x != 'DEPRECATED'
]
# shipit tallies
ansible_shipits = 0
maintainer_shipits = 0
community_shipits = 0
shipit_actors = []
for event in iw.history.history:
if event['event'] not in ['commented', 'committed']:
continue
if event['actor'] in botnames:
continue
# commits reset the counters
if event['event'] == 'committed':
ansible_shipits = 0
maintainer_shipits = 0
community_shipits = 0
shipit_actors = []
continue
actor = event['actor']
body = event['body']
# ansible shipits
if actor in core_team:
if 'shipit' in body or '+1' in body or 'LGTM' in body:
logging.info('%s shipit' % actor)
if actor not in shipit_actors:
ansible_shipits += 1
shipit_actors.append(actor)
continue
# maintainer shipits
if actor in maintainers:
if 'shipit' in body or '+1' in body or 'LGTM' in body:
logging.info('%s shipit' % actor)
if actor not in shipit_actors:
maintainer_shipits += 1
shipit_actors.append(actor)
continue
# community shipits
if actor in community:
if 'shipit' in body or '+1' in body or 'LGTM' in body:
logging.info('%s shipit' % actor)
if actor not in shipit_actors:
community_shipits += 1
shipit_actors.append(actor)
continue
# submitters should count if they are maintainers/community
if iw.submitter in maintainers:
if iw.submitter not in shipit_actors:
maintainer_shipits += 1
shipit_actors.append(iw.submitter)
elif iw.submitter in community:
if iw.submitter not in shipit_actors:
community_shipits += 1
shipit_actors.append(iw.submitter)
nmeta['shipit_count_community'] = community_shipits
nmeta['shipit_count_maintainer'] = maintainer_shipits
nmeta['shipit_count_ansible'] = ansible_shipits
nmeta['shipit_actors'] = shipit_actors
nmeta['community_usernames'] = sorted(community)
if (community_shipits + maintainer_shipits + ansible_shipits) > 1:
nmeta['shipit'] = True
elif meta['is_new_module'] or \
(len(maintainers) == 1 and maintainer_shipits == 1):
if community:
bpc = iw.history.get_boilerplate_comments()
if 'community_shipit_notify' not in bpc:
nmeta['notify_community_shipit'] = True
logging.info('total shipits: %s' %
(community_shipits + maintainer_shipits + ansible_shipits))
return nmeta
def get_shipit_facts(issuewrapper, inmeta, module_indexer, core_team=[], botnames=[]):
""" Count shipits by maintainers/community/other """
# supershipit - maintainers with isolated commit access
# maintainers - people who maintain this file/module
# community - people who maintain file(s) in the same directory
# other - anyone else who comments with shipit/+1/LGTM
meta = inmeta.copy()
iw = issuewrapper
nmeta = {
u'shipit': False,
u'supershipit': False,
u'owner_pr': False,
u'shipit_ansible': False,
u'shipit_community': False,
u'shipit_count_other': False,
u'shipit_count_community': False,
u'shipit_count_maintainer': False,
u'shipit_count_ansible': False,
u'shipit_count_vtotal': False,
u'shipit_actors': None,
u'supershipit_actors': None,
u'community_usernames': [],
u'notify_community_shipit': False,
u'is_rebuild_merge': False,
}
if not iw.is_pullrequest():
return nmeta
# https://github.com/ansible/ansibullbot/issues/1147
meta[u'component_matches'] = [
x for x in meta.get(u'component_matches', [])
if not x[u'repo_filename'].startswith(u'changelogs/fragments/')
]
module_utils_files_owned = 0 # module_utils files for which submitter is maintainer
if meta[u'is_module_util']:
for f in iw.files:
if f.startswith(u'lib/ansible/module_utils') and f in module_indexer.botmeta[u'files']:
maintainers = module_indexer.botmeta[u'files'][f].get(u'maintainers', [])
if maintainers and (iw.submitter in maintainers):
module_utils_files_owned += 1
modules_files_owned = 0
if not meta[u'is_new_module']:
for f in iw.files:
if f.startswith(u'lib/ansible/modules') and iw.submitter in meta[u'component_maintainers']:
modules_files_owned += 1
nmeta[u'owner_pr'] = modules_files_owned + module_utils_files_owned == len(iw.files)
# https://github.com/ansible/ansibullbot/issues/722
if iw.wip:
logging.debug(u'WIP PRs do not get shipits')
return nmeta
if meta[u'is_needs_revision'] or meta[u'is_needs_rebase']:
logging.debug(u'PRs with needs_revision or needs_rebase label do not get shipits')
return nmeta
supershipiteers_byfile = {}
supershipiteers_byuser = {}
for cm in meta.get('component_matches', []):
_ss = cm.get(u'supershipit', [])
supershipiteers_byfile[cm[u'repo_filename']] = _ss[:]
for ss in _ss:
if ss not in supershipiteers_byuser:
supershipiteers_byuser[ss] = []
supershipiteers_byuser[ss].append(cm[u'repo_filename'])
maintainers = meta.get(u'component_maintainers', [])
maintainers = \
ModuleIndexer.replace_ansible(
maintainers,
core_team,
bots=botnames
)
# community is the other maintainers in the same namespace
community = meta.get(u'component_namespace_maintainers', [])
community = [x for x in community if x != u'ansible' and
x not in core_team and
x != u'DEPRECATED']
# shipit tallies
ansible_shipits = 0
maintainer_shipits = 0
community_shipits = 0
other_shipits = 0
shipit_actors = []
shipit_actors_other = []
supershipiteers_voted = set()
rebuild_merge = False
for event in iw.history.history:
if event[u'event'] not in [u'commented', u'committed', u'review_approved', u'review_comment']:
continue
if event[u'actor'] in botnames:
continue
# commits reset the counters
if event[u'event'] == u'committed':
logging.info(event)
ansible_shipits = 0
maintainer_shipits = 0
community_shipits = 0
other_shipits = 0
shipit_actors = []
shipit_actors_other = []
supershipiteers_voted = set()
rebuild_merge = False
logging.info('commit detected, resetting shipit tallies')
continue
actor = event[u'actor']
body = event.get(u'body', u'')
body = body.strip()
if not is_approval(body):
continue
if actor in core_team and is_rebuild_merge(body):
rebuild_merge = True
logging.info(u'%s shipit [rebuild_merge]' % actor)
else:
logging.info(u'%s shipit' % actor)
# super shipits
if actor in supershipiteers_byuser:
supershipiteers_voted.add(actor)
# ansible shipits
if actor in core_team:
if actor not in shipit_actors:
ansible_shipits += 1
shipit_actors.append(actor)
continue
# maintainer shipits
if actor in maintainers:
if actor not in shipit_actors:
maintainer_shipits += 1
shipit_actors.append(actor)
continue
# community shipits
if actor in community:
if actor not in shipit_actors:
community_shipits += 1
shipit_actors.append(actor)
continue
# other shipits
if actor not in shipit_actors_other:
other_shipits += 1
shipit_actors_other.append(actor)
continue
# submitters should count if they are core team/maintainers/community
if iw.submitter in core_team:
if iw.submitter not in shipit_actors:
ansible_shipits += 1
shipit_actors.append(iw.submitter)
elif iw.submitter in maintainers:
if iw.submitter not in shipit_actors:
maintainer_shipits += 1
shipit_actors.append(iw.submitter)
elif iw.submitter in community:
if iw.submitter not in shipit_actors:
community_shipits += 1
shipit_actors.append(iw.submitter)
nmeta[u'shipit_count_other'] = other_shipits
nmeta[u'shipit_count_community'] = community_shipits
nmeta[u'shipit_count_maintainer'] = maintainer_shipits
nmeta[u'shipit_count_ansible'] = ansible_shipits
nmeta[u'shipit_actors'] = shipit_actors
nmeta[u'shipit_actors_other'] = shipit_actors_other
nmeta[u'community_usernames'] = sorted(community)
total = community_shipits + maintainer_shipits + ansible_shipits
nmeta[u'shipit_count_vtotal'] = total + other_shipits
if rebuild_merge:
nmeta['is_rebuild_merge'] = True
# include shipits from other people to push over the edge
if total == 1 and other_shipits > 2:
total += other_shipits
if total > 1 or rebuild_merge:
nmeta[u'shipit'] = True
elif meta[u'is_new_module'] or \
(len(maintainers) == 1 and maintainer_shipits == 1):
# don't notify if there is no maintainer or if submitter is the only namespace maintainer
if set(community) - {iw.submitter}:
bpc = iw.history.get_boilerplate_comments()
bpc = [x[0] for x in bpc]
if u'community_shipit_notify' not in bpc:
nmeta[u'notify_community_shipit'] = True
logging.info(u'total shipits: %s' % total)
# supershipit ...
# if a supershipiteer for each file exists and has blessed the PR
# on the current commit, then override all shipit tallies and get this PR merged
if supershipiteers_voted:
nmeta[u'supershipit_actors'] = list(supershipiteers_voted)
cm_files = [x[u'repo_filename'] for x in meta[u'component_matches']]
ss_files = set()
for ssv in supershipiteers_voted:
for fn in supershipiteers_byuser[ssv]:
ss_files.add(fn)
if sorted(set(cm_files)) == sorted(set(ss_files)):
logging.info(u'supershipit enabled on %s' % iw.html_url)
nmeta[u'supershipit'] = True
nmeta[u'shipit'] = True
else:
for cm_file in sorted(cm_files):
if cm_file not in ss_files:
logging.info('%s is not governed by supershipit' % cm_file)
return nmeta
def main():
pprint(sys.argv)
dest = sys.argv[1]
print('dest: %s' % dest)
# get_valid_labels('ansible/ansible')
# /home/jtanner/.ansibullbot/cache/ansible/ansible/labels.pickle
with open(os.path.expanduser('~/.ansibullbot/cache/ansible/ansible/labels.pickle'), 'rb') as f:
labels = pickle.load(f)
valid_labels = [x.name for x in labels[1]]
FILEMAP_FILENAME = 'FILEMAP.json'
COMPONENTMAP_FILENAME = 'COMPONENTMAP.json'
FI = FileIndexer(
checkoutdir=os.path.expanduser(
'~/.ansibullbot/cache/ansible.files.checkout'
),
cmap=COMPONENTMAP_FILENAME,
)
module_cache_file = '/tmp/mi-modules.json'
if not os.path.isfile(module_cache_file):
module_maintainers = get_maintainers_mapping()
MI = ModuleIndexer(maintainers=module_maintainers)
MI.get_ansible_modules()
with open(module_cache_file, 'wb') as f:
f.write(json.dumps(MI.modules, sort_keys=True, indent=2))
modules = MI.modules
else:
with open(module_cache_file, 'rb') as f:
modules = json.loads(f.read())
macro_teams = {
'Qalthos,gundalow,privateip': 'openswitch',
'Qalthos,ganeshrn,gundalow,privateip,rcarrillocruz,trishnaguha': 'networking',
'GGabriele,jedelman8,mikewiebe,privateip,rahushen,rcarrillocruz,trishnaguha': 'nxos',
'emonty,j2sol,juliakreger,rcarrillocruz,shrews,thingee': 'openstack',
'chrishoffman,manuel-sousa,romanek-adam': 'rabbitmq',
'alikins,barnabycourt,flossware,vritant': 'rhn',
'Qalthos,amitsi,gundalow,privateip': 'netvisor',
'haroldwongms,nitzmahone,tstringer': 'azure',
'dagwieers,jborean93,jhawkesworth': 'windows',
'dagwieers,dav1x,jctanner': 'vmware',
'isharacomix,jrrivers,privateip': 'cumulus',
'chiradeep,giorgos-nikolopoulos': 'netscaler',
'ericsysmin,grastogi23,khaltore': 'avi',
'ghjm,jlaska,matburt,wwitzel3': 'tower',
'hulquest,lmprice,timuster': 'netapp',
}
usermap = {
'mpdehaan': False
}
namemap = {
'Shrews': 'shrews'
}
exclusions = {
'*': ['chouseknecht', 'Java1Guy', 'franckcuny', 'mhite', 'bennojoy', 'risaacson', 'whenrik'],
'network/wakeonlan': ['dagwiers'],
}
removed = get_removed_maintainers()
teams = {}
data = {}
data['files'] = {}
# merge the moduleindexer data
for k,v in modules.items():
fp = v.get('filepath')
if not fp or not fp.startswith('lib/ansible'):
continue
data['files'][k] = {}
if v['_maintainers']:
data['files'][k]['maintainers'] = []
data['files'][k]['maintainers'] = [x for x in v['_maintainers']]
if v['authors']:
if 'maintainers' not in data['files'][k]:
data['files'][k]['maintainers'] = []
data['files'][k]['maintainers'] += v['authors']
data['files'][k]['maintainers'] = sorted(set(data['files'][k]['maintainers']))
# validate each maintainer exists
if 'maintainers' in data['files'][k]:
maintainers = []
for x in data['files'][k]['maintainers']:
if x in exclusions['*']:
continue
if x in namemap:
x = namemap[x]
if x in usermap:
if usermap[x]:
maintainers.append(x)
else:
if x == 'ansible':
usermap['ansible'] = True
maintainers.append(x)
continue
res = requests.get('https://github.com/%s' % x)
if res.status_code == 200:
usermap[x] = True
maintainers.append(x)
else:
usermap[x] = False
data['files'][k]['maintainers'] = sorted(set(maintainers))
if not data['files'][k]['maintainers']:
data['files'][k].pop('maintainers', None)
# merge the removed people
for k,v in removed.items():
k = os.path.join('lib/ansible/modules', k)
v = sorted(set(v))
if k in data['files']:
if 'maintainers' in data['files'][k]:
for vx in v:
if vx in data['files'][k]['maintainers']:
data['files'][k]['maintainers'].remove(vx)
if 'ignored' not in data['files'][k]:
data['files'][k]['ignored'] = []
data['files'][k]['ignored'].append(vx)
if not data['files'][k]['maintainers']:
data['files'][k].pop('maintainers', None)
#import epdb; epdb.st()
# merge the fileindexer data
for k in FI.files:
#if 'contrib/inventory' in k:
# import epdb; epdb.st()
#print(k)
try:
klabels = FI.get_component_labels(valid_labels, [k])
if klabels:
klabels = [x for x in klabels if not x.startswith('c:')]
if not klabels:
continue
if k not in data['files']:
data['files'][k] = {}
if 'labels' not in data['files'][k]:
data['files'][k]['labels'] = []
data['files'][k]['labels'] += klabels
except UnicodeDecodeError:
continue
keywords = FI.get_keywords_for_file(k)
if keywords:
if k not in data['files']:
data['files'][k] = {}
if 'keywords' not in data['files'][k]:
data['files'][k]['keywords'] = []
data['files'][k]['keywords'] += keywords
#import epdb; epdb.st()
'''
# calculate all teams
for k,v in data['files'].items():
if not v.get('maintainers'):
continue
maintainers = sorted(set(v['maintainers']))
key = ','.join(maintainers)
if key not in teams:
teams[key] = []
teams[key].append(k)
# rank and show
steams = sorted(teams, key=len, reverse=True)
for x in steams[0:15]:
if x in macro_teams:
continue
pprint(teams[x])
print(x)
import epdb; epdb.st()
import epdb; epdb.st()
'''
for k,v in data['files'].items():
if not v.get('maintainers'):
continue
maintainers = v.get('maintainers')
for idx,x in enumerate(maintainers):
if x == 'ansible':
maintainers[idx] = '$team_ansible'
if maintainers == ['$team_ansible']:
data['files'][k]['maintainers'] = ' '.join(maintainers)
continue
if len(maintainers) == 1:
data['files'][k]['maintainers'] = ' '.join(maintainers)
continue
mkey = ','.join(sorted(set(maintainers)))
if mkey in macro_teams:
maintainers = ['$team_%s' % macro_teams[mkey]]
data['files'][k]['maintainers'] = ' '.join(maintainers)
else:
# partial matching
match = None
subnames = sorted(set(maintainers))
for sn in subnames:
filtered = [x for x in subnames if x != sn]
fkey = ','.join(filtered)
if fkey in macro_teams:
match = fkey
if match:
to_clear = match.split(',')
maintainers = [x for x in maintainers if x not in to_clear]
data['files'][k]['maintainers'] = ' '.join(maintainers)
# fix deprecations
safe_names = [x for x in FI.files if all(c in string.printable for c in x)]
remove = []
for k,v in data['files'].items():
maintainers = v.get('maintainers')
if maintainers:
if 'DEPRECATED' in data['files'][k]['maintainers']:
data['files'][k].pop('maintainers', None)
data['files'][k]['deprecated'] = True
bn = os.path.basename(k)
if bn.startswith('_') and bn != '__init__.py' and '/modules/' in k:
'''
data['files'][k]['deprecated'] = True
if 'maintainers' in data['files'][k]:
data['files'][k].pop('maintainers', None)
'''
remove.append(k)
# get rid of files no longer in the repo
if k not in safe_names:
remove.append(k)
for x in remove:
data['files'].pop(x, None)
# remove any keys where maintainers == authors
remove = []
for k,v in data['files'].items():
if v.keys() != ['maintainers']:
continue
if v['maintainers'] != modules[k]['authors']:
continue
remove.append(k)
for x in remove:
data['files'].pop(x, None)
#####################################
# add special notifies
#####################################
data['files']['lib/ansible/modules/cloud/amazon/'] = {
'notify': ['willthames']
}
#####################################
# reduce to namespace maintainers
#####################################
groups = {}
for k,v in data['files'].items():
dn = os.path.dirname(k)
if dn not in groups:
groups[dn] = {
'matches': [],
'values': []
}
groups[dn]['matches'].append(k)
if v not in groups[dn]['values']:
groups[dn]['values'].append(v)
for k,v in groups.items():
if not len(v['values']) == 1:
continue
if len(v['matches']) == 1:
continue
#print(k)
#pprint(v)
newk = k + '/'
data['files'][newk] = v['values'][0]
for pf in v['matches']:
data['files'].pop(pf, None)
if newk in removed:
import epdb; epdb.st()
#####################################
# make a sorted dict
#####################################
files = data['files']
data['files'] = OrderedDict()
fkeys = sorted(files.keys())
fkeys = [x.replace('lib/ansible/modules', '$modules') for x in fkeys]
fkeys = sorted(set(fkeys))
for fkey in fkeys:
if fkey.startswith('$modules'):
mkey = fkey.replace('$modules', 'lib/ansible/modules')
data['files'][fkey] = files[mkey]
else:
data['files'][fkey] = files[fkey]
data['macros'] = OrderedDict()
data['macros']['modules'] = 'lib/ansible/modules'
macro_items = macro_teams.items()
macro_items = [[x[1],x[0]] for x in macro_items]
macro_dict ={}
for x in macro_items:
macro_dict[x[0]] = x[1]
data['macros']['team_ansible'] = []
keys = macro_dict.keys()
for k in sorted(keys):
team = macro_dict[k]
team = team.split(',')
if len(team) < 10:
team = " ".join(team)
data['macros']['team_%s' % k] = team
# if maintainers is the only subkey, make the primary value a string
for k,v in data['files'].items():
keys = v.keys()
if keys == ['maintainers']:
if isinstance(v['maintainers'], list):
data['files'][k] = " ".join(v['maintainers'])
else:
data['files'][k] = v['maintainers']
for xk in ['ignored', 'notified', 'maintainers']:
if xk in data['files'][k]:
if not isinstance(data['files'][k][xk], (str, unicode)):
data['files'][k][xk] = " ".join(data['files'][k][xk])
# write it once with ryaml to make it ordered
ryaml = rYAML()
(fo, fn) = tempfile.mkstemp()
with open(fn, 'wb') as f:
ryaml.dump(data, f)
# read it back in
with open(fn, 'rb') as f:
ylines = f.readlines()
phase = None
for idx,x in enumerate(ylines):
x = x.rstrip()
x = x.replace('!!omap', '')
if x.endswith(' {}'):
x = x.replace(' {}', '')
if x.startswith('-'):
x = x.replace('-', ' ', 1)
ylines[idx] = x
if x.startswith(' ') and ':' not in x and '-' not in x:
ylines[idx-1] += ' ' + x.strip()
ylines[idx] = ''
ylines = [x for x in ylines if x.strip()]
ylines = [HEADER] + ylines
with open(dest, 'wb') as f:
f.write('\n'.join(ylines))
def indexer(m_update, m_authors, m_modules, m_content):
module_indexer = ModuleIndexer()
module_indexer.parse_metadata()
module_indexer.set_maintainers()
return module_indexer
def get_shipit_facts(issuewrapper, meta, module_indexer, core_team=[], botnames=[]):
""" Count shipits by maintainers/community/other """
# maintainers - people who maintain this file/module
# community - people who maintain file(s) in the same directory
# other - anyone else who comments with shipit/+1/LGTM
iw = issuewrapper
nmeta = {
u'shipit': False,
u'owner_pr': False,
u'shipit_ansible': False,
u'shipit_community': False,
u'shipit_count_other': False,
u'shipit_count_community': False,
u'shipit_count_maintainer': False,
u'shipit_count_ansible': False,
u'shipit_count_vtotal': False,
u'shipit_actors': None,
u'community_usernames': [],
u'notify_community_shipit': False,
}
if not iw.is_pullrequest():
return nmeta
module_utils_files_owned = 0 # module_utils files for which submitter is maintainer
if meta[u'is_module_util']:
for f in iw.files:
if f.startswith(u'lib/ansible/module_utils') and f in module_indexer.botmeta[u'files']:
maintainers = module_indexer.botmeta[u'files'][f].get(u'maintainers', [])
if maintainers and (iw.submitter in maintainers):
module_utils_files_owned += 1
modules_files_owned = 0
if not meta[u'is_new_module']:
for f in iw.files:
if f.startswith(u'lib/ansible/modules') and iw.submitter in meta[u'component_maintainers']:
modules_files_owned += 1
nmeta[u'owner_pr'] = modules_files_owned + module_utils_files_owned == len(iw.files)
#if not meta['module_match']:
# return nmeta
# https://github.com/ansible/ansibullbot/issues/722
if iw.wip:
logging.debug(u'WIP PRs do not get shipits')
return nmeta
if meta[u'is_needs_revision'] or meta[u'is_needs_rebase']:
logging.debug(u'PRs with needs_revision or needs_rebase label do not get shipits')
return nmeta
maintainers = meta.get(u'component_maintainers', [])
maintainers = \
ModuleIndexer.replace_ansible(
maintainers,
core_team,
bots=botnames
)
# community is the other maintainers in the same namespace
community = meta.get(u'component_namespace_maintainers', [])
community = [x for x in community if x != u'ansible' and
x not in core_team and
x != u'DEPRECATED']
# shipit tallies
ansible_shipits = 0
maintainer_shipits = 0
community_shipits = 0
other_shipits = 0
shipit_actors = []
shipit_actors_other = []
for event in iw.history.history:
if event[u'event'] not in [u'commented', u'committed', u'review_approved', u'review_comment']:
continue
if event[u'actor'] in botnames:
continue
# commits reset the counters
if event[u'event'] == u'committed':
ansible_shipits = 0
maintainer_shipits = 0
community_shipits = 0
other_shipits = 0
shipit_actors = []
shipit_actors_other = []
continue
actor = event[u'actor']
body = event.get(u'body', u'')
body = body.strip()
if not is_approval(body):
continue
logging.info(u'%s shipit' % actor)
# ansible shipits
if actor in core_team:
if actor not in shipit_actors:
ansible_shipits += 1
shipit_actors.append(actor)
continue
# maintainer shipits
if actor in maintainers:
if actor not in shipit_actors:
maintainer_shipits += 1
shipit_actors.append(actor)
continue
# community shipits
if actor in community:
if actor not in shipit_actors:
community_shipits += 1
shipit_actors.append(actor)
continue
# other shipits
if actor not in shipit_actors_other:
other_shipits += 1
shipit_actors_other.append(actor)
continue
# submitters should count if they are core team/maintainers/community
if iw.submitter in core_team:
if iw.submitter not in shipit_actors:
ansible_shipits += 1
shipit_actors.append(iw.submitter)
elif iw.submitter in maintainers:
if iw.submitter not in shipit_actors:
maintainer_shipits += 1
shipit_actors.append(iw.submitter)
elif iw.submitter in community:
if iw.submitter not in shipit_actors:
community_shipits += 1
shipit_actors.append(iw.submitter)
nmeta[u'shipit_count_other'] = other_shipits
nmeta[u'shipit_count_community'] = community_shipits
nmeta[u'shipit_count_maintainer'] = maintainer_shipits
nmeta[u'shipit_count_ansible'] = ansible_shipits
nmeta[u'shipit_actors'] = shipit_actors
nmeta[u'shipit_actors_other'] = shipit_actors_other
nmeta[u'community_usernames'] = sorted(community)
total = community_shipits + maintainer_shipits + ansible_shipits
nmeta[u'shipit_count_vtotal'] = total + other_shipits
# include shipits from other people to push over the edge
if total == 1 and other_shipits > 2:
total += other_shipits
if total > 1:
nmeta[u'shipit'] = True
elif meta[u'is_new_module'] or \
(len(maintainers) == 1 and maintainer_shipits == 1):
# don't notify if there is no maintainer or if submitter is the only namespace maintainer
if set(community) - {iw.submitter}:
bpc = iw.history.get_boilerplate_comments()
bpc = [x[0] for x in bpc]
if u'community_shipit_notify' not in bpc:
nmeta[u'notify_community_shipit'] = True
logging.info(u'total shipits: %s' % total)
return nmeta
def main():
pprint(sys.argv)
dest = sys.argv[1]
print('dest: %s' % dest)
# get_valid_labels('ansible/ansible')
# /home/jtanner/.ansibullbot/cache/ansible/ansible/labels.pickle
with open(os.path.expanduser('~/.ansibullbot/cache/ansible/ansible/labels.pickle'), 'rb') as f:
labels = pickle_load(f)
valid_labels = [x.name for x in labels[1]]
FILEMAP_FILENAME = 'FILEMAP.json'
COMPONENTMAP_FILENAME = 'COMPONENTMAP.json'
FI = FileIndexer(
checkoutdir=os.path.expanduser(
'~/.ansibullbot/cache/ansible.files.checkout'
),
cmap=COMPONENTMAP_FILENAME,
)
module_cache_file = '/tmp/mi-modules.json'
if not os.path.isfile(module_cache_file):
module_maintainers = get_maintainers_mapping()
MI = ModuleIndexer(maintainers=module_maintainers)
MI.get_ansible_modules()
with open(module_cache_file, 'wb') as f:
f.write(json.dumps(MI.modules, sort_keys=True, indent=2))
modules = MI.modules
else:
with open(module_cache_file, 'rb') as f:
modules = json.loads(f.read())
macro_teams = {
'Qalthos,gundalow,privateip': 'openswitch',
'Qalthos,ganeshrn,gundalow,privateip,rcarrillocruz,trishnaguha': 'networking',
'GGabriele,jedelman8,mikewiebe,privateip,rahushen,rcarrillocruz,trishnaguha': 'nxos',
'emonty,j2sol,juliakreger,rcarrillocruz,shrews,thingee': 'openstack',
'chrishoffman,manuel-sousa,romanek-adam': 'rabbitmq',
'alikins,barnabycourt,flossware,vritant': 'rhn',
'Qalthos,amitsi,gundalow,privateip': 'netvisor',
'haroldwongms,nitzmahone,tstringer': 'azure',
'dagwieers,jborean93,jhawkesworth': 'windows',
'dagwieers,dav1x,jctanner': 'vmware',
'isharacomix,jrrivers,privateip': 'cumulus',
'chiradeep,giorgos-nikolopoulos': 'netscaler',
'ericsysmin,grastogi23,khaltore': 'avi',
'ghjm,jlaska,matburt,wwitzel3': 'tower',
'hulquest,lmprice,timuster': 'netapp',
}
usermap = {
'mpdehaan': False
}
namemap = {
'Shrews': 'shrews'
}
exclusions = {
'*': ['chouseknecht', 'Java1Guy', 'franckcuny', 'mhite', 'bennojoy', 'risaacson', 'whenrik'],
'network/wakeonlan': ['dagwiers'],
}
removed = get_removed_maintainers()
teams = {}
data = {}
data['files'] = {}
# merge the moduleindexer data
for k,v in modules.items():
fp = v.get('filepath')
if not fp or not fp.startswith('lib/ansible'):
continue
data['files'][k] = {}
if v['_maintainers']:
data['files'][k]['maintainers'] = []
data['files'][k]['maintainers'] = [x for x in v['_maintainers']]
if v['authors']:
if 'maintainers' not in data['files'][k]:
data['files'][k]['maintainers'] = []
data['files'][k]['maintainers'] += v['authors']
data['files'][k]['maintainers'] = sorted(set(data['files'][k]['maintainers']))
# validate each maintainer exists
if 'maintainers' in data['files'][k]:
maintainers = []
for x in data['files'][k]['maintainers']:
if x in exclusions['*']:
continue
if x in namemap:
x = namemap[x]
if x in usermap:
if usermap[x]:
maintainers.append(x)
else:
if x == 'ansible':
usermap['ansible'] = True
maintainers.append(x)
continue
res = requests.get('https://github.com/%s' % x)
if res.status_code == 200:
usermap[x] = True
maintainers.append(x)
else:
usermap[x] = False
data['files'][k]['maintainers'] = sorted(set(maintainers))
if not data['files'][k]['maintainers']:
data['files'][k].pop('maintainers', None)
# merge the removed people
for k,v in removed.items():
k = os.path.join('lib/ansible/modules', k)
v = sorted(set(v))
if k in data['files']:
if 'maintainers' in data['files'][k]:
for vx in v:
if vx in data['files'][k]['maintainers']:
data['files'][k]['maintainers'].remove(vx)
if 'ignored' not in data['files'][k]:
data['files'][k]['ignored'] = []
data['files'][k]['ignored'].append(vx)
if not data['files'][k]['maintainers']:
data['files'][k].pop('maintainers', None)
#import epdb; epdb.st()
# merge the fileindexer data
for k in FI.files:
#if 'contrib/inventory' in k:
# import epdb; epdb.st()
#print(k)
try:
klabels = FI.get_component_labels(valid_labels, [k])
if klabels:
klabels = [x for x in klabels if not x.startswith('c:')]
if not klabels:
continue
if k not in data['files']:
data['files'][k] = {}
if 'labels' not in data['files'][k]:
data['files'][k]['labels'] = []
data['files'][k]['labels'] += klabels
except UnicodeDecodeError:
continue
keywords = FI.get_keywords_for_file(k)
if keywords:
if k not in data['files']:
data['files'][k] = {}
if 'keywords' not in data['files'][k]:
data['files'][k]['keywords'] = []
data['files'][k]['keywords'] += keywords
#import epdb; epdb.st()
'''
# calculate all teams
for k,v in data['files'].items():
if not v.get('maintainers'):
continue
maintainers = sorted(set(v['maintainers']))
key = ','.join(maintainers)
if key not in teams:
teams[key] = []
teams[key].append(k)
# rank and show
steams = sorted(teams, key=len, reverse=True)
for x in steams[0:15]:
if x in macro_teams:
continue
pprint(teams[x])
print(x)
import epdb; epdb.st()
import epdb; epdb.st()
'''
for k,v in data['files'].items():
if not v.get('maintainers'):
continue
maintainers = v.get('maintainers')
for idx,x in enumerate(maintainers):
if x == 'ansible':
maintainers[idx] = '$team_ansible'
if maintainers == ['$team_ansible']:
data['files'][k]['maintainers'] = ' '.join(maintainers)
continue
if len(maintainers) == 1:
data['files'][k]['maintainers'] = ' '.join(maintainers)
continue
mkey = ','.join(sorted(set(maintainers)))
if mkey in macro_teams:
maintainers = ['$team_%s' % macro_teams[mkey]]
data['files'][k]['maintainers'] = ' '.join(maintainers)
else:
# partial matching
match = None
subnames = sorted(set(maintainers))
for sn in subnames:
filtered = [x for x in subnames if x != sn]
fkey = ','.join(filtered)
if fkey in macro_teams:
match = fkey
if match:
to_clear = match.split(',')
maintainers = [x for x in maintainers if x not in to_clear]
data['files'][k]['maintainers'] = ' '.join(maintainers)
# fix deprecations
safe_names = [x for x in FI.files if all(c in string.printable for c in x)]
remove = []
for k,v in data['files'].items():
maintainers = v.get('maintainers')
if maintainers:
if 'DEPRECATED' in data['files'][k]['maintainers']:
data['files'][k].pop('maintainers', None)
data['files'][k]['deprecated'] = True
bn = os.path.basename(k)
if bn.startswith('_') and bn != '__init__.py' and '/modules/' in k:
'''
data['files'][k]['deprecated'] = True
if 'maintainers' in data['files'][k]:
data['files'][k].pop('maintainers', None)
'''
remove.append(k)
# get rid of files no longer in the repo
if k not in safe_names:
remove.append(k)
for x in remove:
data['files'].pop(x, None)
# remove any keys where maintainers == authors
remove = []
for k,v in data['files'].items():
if v.keys() != ['maintainers']:
continue
if v['maintainers'] != modules[k]['authors']:
continue
remove.append(k)
for x in remove:
data['files'].pop(x, None)
#####################################
# add special notifies
#####################################
data['files']['lib/ansible/modules/cloud/amazon/'] = {
'notify': ['willthames']
}
#####################################
# reduce to namespace maintainers
#####################################
groups = {}
for k,v in data['files'].items():
dn = os.path.dirname(k)
if dn not in groups:
groups[dn] = {
'matches': [],
'values': []
}
groups[dn]['matches'].append(k)
if v not in groups[dn]['values']:
groups[dn]['values'].append(v)
for k,v in groups.items():
if not len(v['values']) == 1:
continue
if len(v['matches']) == 1:
continue
#print(k)
#pprint(v)
newk = k + '/'
data['files'][newk] = v['values'][0]
for pf in v['matches']:
data['files'].pop(pf, None)
if newk in removed:
import epdb; epdb.st()
#####################################
# make a sorted dict
#####################################
files = data['files']
data['files'] = OrderedDict()
fkeys = sorted(files.keys())
fkeys = [x.replace('lib/ansible/modules', '$modules') for x in fkeys]
fkeys = sorted(set(fkeys))
for fkey in fkeys:
if fkey.startswith('$modules'):
mkey = fkey.replace('$modules', 'lib/ansible/modules')
data['files'][fkey] = files[mkey]
else:
data['files'][fkey] = files[fkey]
data['macros'] = OrderedDict()
data['macros']['modules'] = 'lib/ansible/modules'
macro_items = macro_teams.items()
macro_items = [[x[1],x[0]] for x in macro_items]
macro_dict ={}
for x in macro_items:
macro_dict[x[0]] = x[1]
data['macros']['team_ansible'] = []
keys = macro_dict.keys()
for k in sorted(keys):
team = macro_dict[k]
team = team.split(',')
if len(team) < 10:
team = " ".join(team)
data['macros']['team_%s' % k] = team
# if maintainers is the only subkey, make the primary value a string
for k,v in data['files'].items():
keys = v.keys()
if keys == ['maintainers']:
if isinstance(v['maintainers'], list):
data['files'][k] = " ".join(v['maintainers'])
else:
data['files'][k] = v['maintainers']
for xk in ['ignored', 'notified', 'maintainers']:
if xk in data['files'][k]:
if not isinstance(data['files'][k][xk], (str, unicode)):
data['files'][k][xk] = " ".join(data['files'][k][xk])
# write it once with ryaml to make it ordered
ryaml = rYAML()
(fo, fn) = tempfile.mkstemp()
with open(fn, 'wb') as f:
ryaml.dump(data, f)
# read it back in
with open(fn, 'rb') as f:
ylines = f.readlines()
phase = None
for idx,x in enumerate(ylines):
x = x.rstrip()
x = x.replace('!!omap', '')
if x.endswith(' {}'):
x = x.replace(' {}', '')
if x.startswith('-'):
x = x.replace('-', ' ', 1)
ylines[idx] = x
if x.startswith(' ') and ':' not in x and '-' not in x:
ylines[idx-1] += ' ' + x.strip()
ylines[idx] = ''
ylines = [x for x in ylines if x.strip()]
ylines = [HEADER] + ylines
with open(dest, 'wb') as f:
f.write('\n'.join(ylines))
def get_shipit_facts(issuewrapper, meta, module_indexer, core_team=[], botnames=[]):
""" Count shipits by maintainers/community/other """
# maintainers - people who maintain this file/module
# community - people who maintain file(s) in the same directory
# other - anyone else who comments with shipit/+1/LGTM
iw = issuewrapper
nmeta = {
'shipit': False,
'owner_pr': False,
'shipit_ansible': False,
'shipit_community': False,
'shipit_count_other': False,
'shipit_count_community': False,
'shipit_count_maintainer': False,
'shipit_count_ansible': False,
'shipit_actors': None,
'community_usernames': [],
'notify_community_shipit': False,
}
if not iw.is_pullrequest():
return nmeta
if not meta['module_match']:
return nmeta
maintainers = meta['module_match']['maintainers']
maintainers = \
ModuleIndexer.replace_ansible(
maintainers,
core_team,
bots=botnames
)
if not meta['is_new_module'] and iw.submitter in maintainers:
nmeta['owner_pr'] = True
# community is the other maintainers in the same namespace
mnamespace = meta['module_match']['namespace']
community = \
module_indexer.get_maintainers_for_namespace(mnamespace)
community = [x for x in community if x != 'ansible' and
x not in core_team and
x != 'DEPRECATED']
# shipit tallies
ansible_shipits = 0
maintainer_shipits = 0
community_shipits = 0
other_shipits = 0
shipit_actors = []
shipit_actors_other = []
for event in iw.history.history:
if event['event'] not in ['commented', 'committed']:
continue
if event['actor'] in botnames:
continue
# commits reset the counters
if event['event'] == 'committed':
ansible_shipits = 0
maintainer_shipits = 0
community_shipits = 0
other_shipits = 0
shipit_actors = []
shipit_actors_other = []
continue
actor = event['actor']
body = event['body']
body = body.strip()
if 'shipit' not in body and '+1' not in body and 'LGTM' not in body:
continue
logging.info('%s shipit' % actor)
# ansible shipits
if actor in core_team:
if actor not in shipit_actors:
ansible_shipits += 1
shipit_actors.append(actor)
continue
# maintainer shipits
if actor in maintainers:
if actor not in shipit_actors:
maintainer_shipits += 1
shipit_actors.append(actor)
continue
# community shipits
if actor in community:
if actor not in shipit_actors:
community_shipits += 1
shipit_actors.append(actor)
continue
# other shipits
if actor not in shipit_actors_other:
other_shipits += 1
shipit_actors_other.append(actor)
continue
# submitters should count if they are maintainers/community
if iw.submitter in maintainers:
if iw.submitter not in shipit_actors:
maintainer_shipits += 1
shipit_actors.append(iw.submitter)
elif iw.submitter in community:
if iw.submitter not in shipit_actors:
community_shipits += 1
shipit_actors.append(iw.submitter)
nmeta['shipit_count_other'] = other_shipits
nmeta['shipit_count_community'] = community_shipits
nmeta['shipit_count_maintainer'] = maintainer_shipits
nmeta['shipit_count_ansible'] = ansible_shipits
nmeta['shipit_actors'] = shipit_actors
nmeta['shipit_actors_other'] = shipit_actors_other
nmeta['community_usernames'] = sorted(community)
total = community_shipits + maintainer_shipits + ansible_shipits
# include shipits from other people to push over the edge
if total == 1 and other_shipits > 2:
total += other_shipits
if total > 1:
nmeta['shipit'] = True
elif meta['is_new_module'] or \
(len(maintainers) == 1 and maintainer_shipits == 1):
if community:
bpc = iw.history.get_boilerplate_comments()
if 'community_shipit_notify' not in bpc:
nmeta['notify_community_shipit'] = True
logging.info('total shipits: %s' % total)
return nmeta
