def __init__(self, verbose=None, github_user=None, github_pass=None,
github_token=None, github_repo=None, number=None,
start_at=None, always_pause=False, force=False, safe_force=False,
dry_run=False, no_since=False):
self.verbose = verbose
self.github_user = github_user
self.github_pass = github_pass
self.github_token = github_token
self.github_repo = github_repo
self.number = number
self.start_at = start_at
self.always_pause = always_pause
self.force = force
self.safe_force = safe_force
self.dry_run = dry_run
self.no_since = no_since
self.issue = None
self.maintainers = {}
self.module_maintainers = []
self.actions = {
'newlabel': [],
'unlabel': [],
'comments': [],
'close': False,
}
# set the cache dir
self.cachedir = '~/.ansibullbot/cache'
if self.github_repo == 'ansible':
self.cachedir += '/ansible/ansible/'
else:
self.cachedir += '/ansible/ansible-modules-%s/' % self.github_repo
self.cachedir += 'issues'
self.cachedir = os.path.expanduser(self.cachedir)
if not os.path.isdir(self.cachedir):
os.makedirs(self.cachedir)
print("Initializing AnsibleVersionIndexer")
self.version_indexer = AnsibleVersionIndexer()
#import epdb; epdb.st()
print("Initializing ModuleIndexer")
self.module_indexer = ModuleIndexer()
self.module_indexer.get_ansible_modules()
print("Initializing FileIndexer")
self.file_indexer = FileIndexer()
self.file_indexer.get_files()
print("Getting ansible members")
self.ansible_members = self.get_ansible_members()
print("Getting valid labels")
self.valid_labels = self.get_valid_labels()
# processed metadata
self.meta = {}
def __init__(self, verbose=None, github_user=None, github_pass=None,
github_token=None, github_repo=None, number=None,
start_at=None, always_pause=False, force=False, safe_force=False,
dry_run=False, no_since=False):
self.verbose = verbose
self.github_user = github_user
self.github_pass = github_pass
self.github_token = github_token
self.github_repo = github_repo
self.number = number
self.start_at = start_at
self.always_pause = always_pause
self.force = force
self.safe_force = safe_force
self.dry_run = dry_run
self.no_since = no_since
self.issue = None
self.maintainers = {}
self.module_maintainers = []
self.actions = {
'newlabel': [],
'unlabel': [],
'comments': [],
'close': False,
}
# set the cache dir
self.cachedir = '~/.ansibullbot/cache'
if self.github_repo == 'ansible':
self.cachedir += '/ansible/ansible/'
else:
self.cachedir += '/ansible/ansible-modules-%s/' % self.github_repo
self.cachedir += 'issues'
self.cachedir = os.path.expanduser(self.cachedir)
if not os.path.isdir(self.cachedir):
os.makedirs(self.cachedir)
print("Initializing AnsibleVersionIndexer")
self.version_indexer = AnsibleVersionIndexer()
#import epdb; epdb.st()
print("Initializing ModuleIndexer")
self.module_indexer = ModuleIndexer()
self.module_indexer.get_ansible_modules()
print("Initializing FileIndexer")
self.file_indexer = FileIndexer()
self.file_indexer.get_files()
print("Getting ansible members")
self.ansible_members = self.get_ansible_members()
print("Getting valid labels")
self.valid_labels = self.get_valid_labels()
# processed metadata
self.meta = {}
class DefaultTriager(object):
BOTLIST = ['gregdek', 'robynbergeron', 'ansibot']
VALID_ISSUE_TYPES = ['bug report', 'feature idea', 'documentation report']
IGNORE_LABELS = [
"aws","azure","cloud",
"feature_pull_request",
"feature_idea",
"bugfix_pull_request",
"bug_report",
"docs_pull_request",
"docs_report",
"in progress",
"docs_pull_request",
"easyfix",
"pending_action",
"gce",
"python3",
"P1","P2","P3","P4",
]
def __init__(self, verbose=None, github_user=None, github_pass=None,
github_token=None, github_repo=None, number=None,
start_at=None, always_pause=False, force=False, safe_force=False,
dry_run=False, no_since=False):
self.verbose = verbose
self.github_user = github_user
self.github_pass = github_pass
self.github_token = github_token
self.github_repo = github_repo
self.number = number
self.start_at = start_at
self.always_pause = always_pause
self.force = force
self.safe_force = safe_force
self.dry_run = dry_run
self.no_since = no_since
self.issue = None
self.maintainers = {}
self.module_maintainers = []
self.actions = {
'newlabel': [],
'unlabel': [],
'comments': [],
'close': False,
}
# set the cache dir
self.cachedir = '~/.ansibullbot/cache'
if self.github_repo == 'ansible':
self.cachedir += '/ansible/ansible/'
else:
self.cachedir += '/ansible/ansible-modules-%s/' % self.github_repo
self.cachedir += 'issues'
self.cachedir = os.path.expanduser(self.cachedir)
if not os.path.isdir(self.cachedir):
os.makedirs(self.cachedir)
print("Initializing AnsibleVersionIndexer")
self.version_indexer = AnsibleVersionIndexer()
#import epdb; epdb.st()
print("Initializing ModuleIndexer")
self.module_indexer = ModuleIndexer()
self.module_indexer.get_ansible_modules()
print("Initializing FileIndexer")
self.file_indexer = FileIndexer()
self.file_indexer.get_files()
print("Getting ansible members")
self.ansible_members = self.get_ansible_members()
print("Getting valid labels")
self.valid_labels = self.get_valid_labels()
# processed metadata
self.meta = {}
def _process(self, usecache=True):
'''Do some initial processing of the issue'''
# clear all actions
self.actions = {
'newlabel': [],
'unlabel': [],
'comments': [],
'close': False,
}
# clear module maintainers
self.module_maintainers = []
# print some general info about the Issue to be processed
print("\n")
print("Issue #%s [%s]: %s" % (self.issue.number, self.icount,
(self.issue.instance.title).encode('ascii','ignore')))
print("%s" % self.issue.instance.html_url)
print("Created at %s" % self.issue.instance.created_at)
print("Updated at %s" % self.issue.instance.updated_at)
# get the template data
self.template_data = self.issue.get_template_data()
# was the issue type defined correctly?
issue_type_defined = False
issue_type_valid = False
issue_type = False
if 'issue type' in self.template_data:
issue_type_defined = True
issue_type = self.template_data['issue type']
if issue_type.lower() in self.VALID_ISSUE_TYPES:
issue_type_valid = True
self.meta['issue_type_defined'] = issue_type_defined
self.meta['issue_type_valid'] = issue_type_valid
self.meta['issue_type'] = issue_type
if self.meta['issue_type_valid']:
self.meta['issue_type_label'] = self.issue_type_to_label(issue_type)
else:
self.meta['issue_type_label'] = None
# What is the ansible version?
self.ansible_version = self.get_ansible_version()
if not isinstance(self.debug, bool):
self.debug('version: %s' % self.ansible_version)
self.ansible_label_version = self.get_ansible_version_major_minor()
if not isinstance(self.debug, bool):
self.debug('lversion: %s' % self.ansible_label_version)
# was component specified?
component_defined = 'component name' in self.template_data
self.meta['component_defined'] = component_defined
# extract the component
component = self.template_data.get('component name', None)
# save the real name
if self.github_repo != 'ansible':
self.match = self.module_indexer.find_match(component) or {}
else:
self.match = self.module_indexer.find_match(component, exact=True) or {}
self.module = self.match.get('name', None)
# check if component is a known module
component_isvalid = self.module_indexer.is_valid(component)
self.meta['component_valid'] = component_isvalid
# smart match modules (only on module repos)
if not component_isvalid and self.github_repo != 'ansible' and not self.match:
#smatch = self.smart_match_module()
if hasattr(self, 'meta'):
self.meta['fuzzy_match_called'] = True
kwargs = dict(
repo=self.github_repo,
title=self.issue.instance.title,
component=self.template_data.get('component name')
)
smatch = self.module_indexer.fuzzy_match(**kwargs)
if self.module_indexer.is_valid(smatch):
self.module = smatch
component = smatch
self.match = self.module_indexer.find_match(smatch)
component_isvalid = self.module_indexer.is_valid(component)
self.meta['component_valid'] = component_isvalid
# Allow globs for module groups
# https://github.com/ansible/ansible-modules-core/issues/3831
craw = self.template_data.get('component_raw')
if self.module_indexer.is_multi(craw):
self.meta['multiple_components'] = True
# get all of the matches
self.matches = self.module_indexer.multi_match(craw)
if self.matches:
# get maintainers for all of the matches
mmap = {}
for match in self.matches:
key = match['filename']
mmap[key] = self.get_maintainers_by_match(match)
# is there a match that represents all included maintainers?
mtuples = [x[1] for x in mmap.items()]
umtuples = [list(x) for x in set(tuple(x) for x in mtuples)]
all_maintainers = []
for mtup in umtuples:
for x in mtup:
if x not in all_maintainers:
all_maintainers.append(x)
best_match = None
for k,v in mmap.iteritems():
if sorted(set(v)) == sorted(set(all_maintainers)):
best_match = k
break
if best_match:
self.match = self.module_indexer.find_match(best_match)
else:
# there's no good match that would include all maintainers
# just skip multi-module processing for now since the rest
# of the code doesn't know what to do with it.
if not isinstance(self.debug, bool):
self.debug('multi-match maintainers: %s' % umtuples)
#print(craw)
#import epdb; epdb.st()
pass
else:
self.meta['multiple_components'] = False
# set the maintainer(s)
self.module_maintainer = [x for x in self.get_module_maintainers()]
self.meta['module_maintainers'] = self.module_maintainer
# fixme: too many places where the module is set
if self.match:
self.module = self.match['name']
# Helper to fix issue descriptions ...
DF = DescriptionFixer(self.issue, self.module_indexer, self.match)
self.issue.new_description = DF.new_description
def _connect(self):
"""Connects to GitHub's API"""
return Github(login_or_token=self.github_token or self.github_user,
password=self.github_pass)
#return GithubWrapper(
# login_or_token=self.github_token or self.github_user,
# password=self.github_pass
# )
def _get_repo_path(self):
if self.github_repo in ['core', 'extras']:
return "ansible/ansible-modules-%s" % self.github_repo
else:
return "ansible/%s" % self.github_repo
def is_pr(self, issue):
if '/pull/' in issue.html_url:
return True
else:
return False
def is_issue(self, issue):
return not self.is_pr(issue)
@ratecheck()
def get_ansible_members(self):
ansible_members = []
update = False
write_cache = False
now = self.get_current_time()
org = self._connect().get_organization("ansible")
cachedir = self.cachedir
if cachedir.endswith('/issues'):
cachedir = os.path.dirname(cachedir)
cachefile = os.path.join(cachedir, 'members.pickle')
if not os.path.isdir(cachedir):
os.makedirs(cachedir)
if os.path.isfile(cachefile):
with open(cachefile, 'rb') as f:
mdata = pickle.load(f)
ansible_members = mdata[1]
if mdata[0] < org.updated_at:
update = True
else:
update = True
write_cache = True
if update:
members = org.get_members()
ansible_members = [x.login for x in members]
# save the data
if write_cache:
mdata = [now, ansible_members]
with open(cachefile, 'wb') as f:
pickle.dump(mdata, f)
#import epdb; epdb.st()
return ansible_members
@ratecheck()
def get_valid_labels(self):
# use the repo wrapper to enable caching+updating
self.gh = self._connect()
self.ghw = GithubWrapper(self.gh)
self.repo = self.ghw.get_repo(self._get_repo_path())
vlabels = []
for vl in self.repo.get_labels():
vlabels.append(vl.name)
return vlabels
def _get_maintainers(self, usecache=True):
"""Reads all known maintainers from files and their owner namespace"""
if not self.maintainers or not usecache:
for repo in ['core', 'extras']:
f = open(MAINTAINERS_FILES[repo])
for line in f:
owner_space = (line.split(': ')[0]).strip()
maintainers_string = (line.split(': ')[-1]).strip()
self.maintainers[owner_space] = maintainers_string.split(' ')
f.close()
# meta is special
self.maintainers['meta'] = ['ansible']
return self.maintainers
def debug(self, msg=""):
"""Prints debug message if verbosity is given"""
if self.verbose:
print("Debug: " + msg)
def get_ansible_version(self):
aversion = None
rawdata = self.template_data.get('ansible version', '')
if rawdata:
aversion = self.version_indexer.strip_ansible_version(rawdata)
if not aversion or aversion == 'devel':
aversion = self.version_indexer.ansible_version_by_date(self.issue.instance.created_at)
if aversion:
if aversion.endswith('.'):
aversion += '0'
# re-run for versions ending with .x
if aversion:
if aversion.endswith('.x'):
aversion = self.version_indexer.strip_ansible_version(aversion)
#import epdb; epdb.st()
if self.version_indexer.is_valid_version(aversion) and aversion != None:
return aversion
else:
# try to go through the submitter's comments and look for the
# first one that specifies a valid version
cversion = None
for comment in self.issue.current_comments:
if comment.user.login != self.issue.instance.user.login:
continue
xver = self.version_indexer.strip_ansible_version(comment.body)
if self.version_indexer.is_valid_version(xver):
cversion = xver
break
# use the comment version
aversion = cversion
return aversion
def get_ansible_version_major_minor(self):
return self.version_indexer.get_major_minor(self.ansible_version)
def get_maintainers_by_match(self, match):
module_maintainers = []
maintainers = self._get_maintainers()
if match['name'] in maintainers:
module_maintainers = maintainers[match['name']]
elif match['repo_filename'] in maintainers:
module_maintainers = maintainers[match['repo_filename']]
elif (match['deprecated_filename']) in maintainers:
module_maintainers = maintainers[match['deprecated_filename']]
elif match['namespaced_module'] in maintainers:
module_maintainers = maintainers[match['namespaced_module']]
elif match['fulltopic'] in maintainers:
module_maintainers = maintainers[match['fulltopic']]
elif (match['topic'] + '/') in maintainers:
module_maintainers = maintainers[match['topic'] + '/']
else:
pass
# Fallback to using the module author(s)
if not module_maintainers and self.match:
if self.match['authors']:
module_maintainers = [x for x in self.match['authors']]
#import epdb; epdb.st()
return module_maintainers
def get_module_maintainers(self, expand=True, usecache=True):
"""Returns the list of maintainers for the current module"""
# expand=False means don't use cache and don't expand the 'ansible' group
if self.module_maintainers and usecache:
return self.module_maintainers
module_maintainers = []
module = self.module
if not module:
return module_maintainers
if not self.module_indexer.is_valid(module):
return module_maintainers
if self.match:
mdata = self.match
else:
mdata = self.module_indexer.find_match(module)
if mdata['repository'] != self.github_repo:
# this was detected and handled in the process loop
pass
# get cached or non-cached maintainers list
if not expand:
maintainers = self._get_maintainers(usecache=False)
else:
maintainers = self._get_maintainers()
if mdata['name'] in maintainers:
module_maintainers = maintainers[mdata['name']]
elif mdata['repo_filename'] in maintainers:
module_maintainers = maintainers[mdata['repo_filename']]
elif (mdata['deprecated_filename']) in maintainers:
module_maintainers = maintainers[mdata['deprecated_filename']]
elif mdata['namespaced_module'] in maintainers:
module_maintainers = maintainers[mdata['namespaced_module']]
elif mdata['fulltopic'] in maintainers:
module_maintainers = maintainers[mdata['fulltopic']]
elif (mdata['topic'] + '/') in maintainers:
module_maintainers = maintainers[mdata['topic'] + '/']
else:
pass
# Fallback to using the module author(s)
if not module_maintainers and self.match:
if self.match['authors']:
module_maintainers = [x for x in self.match['authors']]
# need to set the no maintainer template or assume ansible?
if not module_maintainers and self.module and self.match:
#import epdb; epdb.st()
pass
#import epdb; epdb.st()
return module_maintainers
def get_current_labels(self):
"""Pull the list of labels on this Issue"""
if not self.current_labels:
labels = self.issue.instance.labels
for label in labels:
self.current_labels.append(label.name)
return self.current_labels
def run(self):
pass
def create_actions(self):
pass
def component_from_comments(self):
"""Extracts a component name from special comments"""
# https://github.com/ansible/ansible-modules/core/issues/2618
# comments like: [module: packaging/os/zypper.py] ... ?
component = None
for idx, x in enumerate(self.issue.current_comments):
if '[' in x.body and ']' in x.body and ('module' in x.body or 'component' in x.body or 'plugin' in x.body):
if x.user.login in BOTLIST:
component = x.body.split()[-1]
component = component.replace('[', '')
return component
def has_maintainer_commented(self):
"""Has the maintainer -ever- commented on the issue?"""
commented = False
if self.module_maintainers:
for comment in self.issue.current_comments:
# ignore comments from submitter
if comment.user.login == self.issue.get_submitter():
continue
# "ansible" is special ...
if 'ansible' in self.module_maintainers \
and comment.user.login in self.ansible_members:
commented = True
elif comment.user.login in self.module_maintainers:
commented = True
return commented
def is_maintainer_mentioned(self):
mentioned = False
if self.module_maintainers:
for comment in self.issue.current_comments:
# "ansible" is special ...
if 'ansible' in self.module_maintainers:
for x in self.ansible_members:
if ('@%s' % x) in comment.body:
mentioned = True
break
else:
for x in self.module_maintainers:
if ('@%s' % x) in comment.body:
mentioned = True
break
return mentioned
def get_current_time(self):
#now = datetime.now()
now = datetime.utcnow()
#now = datetime.now(pytz.timezone('US/Pacific'))
#import epdb; epdb.st()
return now
def age_of_last_maintainer_comment(self):
"""How long ago did the maintainer comment?"""
last_comment = None
if self.module_maintainers:
for idx,comment in enumerate(self.issue.current_comments):
# "ansible" is special ...
is_maintainer = False
if 'ansible' in self.module_maintainers \
and comment.user.login in self.ansible_members:
is_maintainer = True
elif comment.user.login in self.module_maintainers:
is_maintainer = True
if is_maintainer:
last_comment = comment
break
if not last_comment:
return -1
else:
now = self.get_current_time()
diff = now - last_comment.created_at
age = diff.days
return age
def is_waiting_on_maintainer(self):
"""Is the issue waiting on the maintainer to comment?"""
waiting = False
if self.module_maintainers:
if not self.issue.current_comments:
return True
creator_last_index = -1
maintainer_last_index = -1
for idx,comment in enumerate(self.issue.current_comments):
if comment.user.login == self.issue.get_submitter():
if creator_last_index == -1 or idx < creator_last_index:
creator_last_index = idx
# "ansible" is special ...
is_maintainer = False
if 'ansible' in self.module_maintainers \
and comment.user.login in self.ansible_members:
is_maintainer = True
elif comment.user.login in self.module_maintainers:
is_maintainer = True
if is_maintainer and \
(maintainer_last_index == -1 or idx < maintainer_last_index):
maintainer_last_index = idx
if creator_last_index == -1 and maintainer_last_index == -1:
waiting = True
elif creator_last_index == -1 and maintainer_last_index > -1:
waiting = False
elif creator_last_index < maintainer_last_index:
waiting = True
return waiting
def keep_current_main_labels(self):
current_labels = self.issue.get_current_labels()
for current_label in current_labels:
if current_label in self.issue.MUTUALLY_EXCLUSIVE_LABELS:
self.issue.add_desired_label(name=current_label)
def add_desired_labels_by_issue_type(self, comments=True):
"""Adds labels by defined issue type"""
issue_type = self.template_data.get('issue type', False)
if issue_type is False:
self.issue.add_desired_label('needs_info')
return
if not issue_type.lower() in self.VALID_ISSUE_TYPES:
# special handling for PRs
if self.issue.instance.pull_request:
mel = [x for x in self.issue.current_labels if x in self.MUTUALLY_EXCLUSIVE_LABELS]
if not mel:
# if only adding new files, assume it is a feature
if self.patch_contains_only_new_files():
issue_type = 'feature pull request'
else:
if not isinstance(self.debug, bool):
self.debug('"%s" was not a valid issue type, adding "needs_info"' % issue_type)
self.issue.add_desired_label('needs_info')
return
else:
if not isinstance(self.debug, bool):
self.debug('"%s" was not a valid issue type, adding "needs_info"' % issue_type)
self.issue.add_desired_label('needs_info')
return
desired_label = issue_type.replace(' ', '_')
desired_label = desired_label.lower()
desired_label = desired_label.replace('documentation', 'docs')
# FIXME - shouldn't have to do this
if desired_label == 'test_pull_request':
desired_label = 'test_pull_requests'
#import epdb; epdb.st()
# is there a mutually exclusive label already?
if desired_label in self.issue.MUTUALLY_EXCLUSIVE_LABELS:
mel = [x for x in self.issue.MUTUALLY_EXCLUSIVE_LABELS \
if x in self.issue.current_labels]
if len(mel) > 0:
return
if desired_label not in self.issue.get_current_labels():
self.issue.add_desired_label(name=desired_label)
if len(self.issue.current_comments) == 0 and comments:
# only set this if no other comments
self.issue.add_desired_comment(boilerplate='issue_new')
def patch_contains_only_new_files(self):
'''Does the PR edit any existing files?'''
oldfiles = False
for x in self.issue.files:
if x.filename.encode('ascii', 'ignore') in self.file_indexer.files:
if not isinstance(self.debug, bool):
self.debug('old file match on %s' % x.filename.encode('ascii', 'ignore'))
oldfiles = True
break
return not oldfiles
def add_desired_labels_by_ansible_version(self):
if not 'ansible version' in self.template_data:
if not isinstance(self.debug, bool):
self.debug(msg="no ansible version section")
self.issue.add_desired_label(name="needs_info")
#self.issue.add_desired_comment(
# boilerplate="issue_missing_data"
#)
return
if not self.template_data['ansible version']:
if not isinstance(self.debug, bool):
self.debug(msg="no ansible version defined")
self.issue.add_desired_label(name="needs_info")
#self.issue.add_desired_comment(
# boilerplate="issue_missing_data"
#)
return
def add_desired_labels_by_namespace(self):
"""Adds labels regarding module namespaces"""
SKIPTOPICS = ['network/basics/']
if not self.match:
return False
'''
if 'component name' in self.template_data and self.match:
if self.match['repository'] != self.github_repo:
self.issue.add_desired_comment(boilerplate='issue_wrong_repo')
'''
for key in ['topic', 'subtopic']:
# ignore networking/basics
if self.match[key] and not self.match['fulltopic'] in SKIPTOPICS:
thislabel = self.issue.TOPIC_MAP.\
get(self.match[key], self.match[key])
if thislabel in self.valid_labels:
self.issue.add_desired_label(thislabel)
def render_comment(self, boilerplate=None):
"""Renders templates into comments using the boilerplate as filename"""
maintainers = self.get_module_maintainers(expand=False)
if not maintainers:
maintainers = ['NO_MAINTAINER_FOUND'] #FIXME - why?
submitter = self.issue.get_submitter()
missing_sections = [x for x in self.issue.REQUIRED_SECTIONS \
if not x in self.template_data \
or not self.template_data.get(x)]
if not self.match and missing_sections:
# be lenient on component name for ansible/ansible
if self.github_repo == 'ansible' and 'component name' in missing_sections:
missing_sections.remove('component name')
#if missing_sections:
# import epdb; epdb.st()
issue_type = self.template_data.get('issue type', None)
if issue_type:
issue_type = issue_type.lower()
correct_repo = self.match.get('repository', None)
template = environment.get_template('%s.j2' % boilerplate)
comment = template.render(maintainers=maintainers,
submitter=submitter,
issue_type=issue_type,
correct_repo=correct_repo,
component_name=self.template_data.get('component name', 'NULL'),
missing_sections=missing_sections)
#import epdb; epdb.st()
return comment
def process_comments(self):
""" Processes ISSUE comments for matching criteria to add labels"""
if not self.github_user in self.BOTLIST:
self.BOTLIST.append(self.github_user)
module_maintainers = self.get_module_maintainers()
comments = self.issue.get_comments()
today = datetime.today()
if not isinstance(self.debug, bool):
self.debug(msg="--- START Processing Comments:")
for idc,comment in enumerate(comments):
if comment.user.login in self.BOTLIST:
if not isinstance(self.debug, bool):
self.debug(msg="%s is in botlist: " % comment.user.login)
time_delta = today - comment.created_at
comment_days_old = time_delta.days
if not isinstance(self.debug, bool):
self.debug(msg="Days since last bot comment: %s" % comment_days_old)
if comment_days_old > 14:
labels = self.issue.desired_labels
if 'pending' not in comment.body:
if self.issue.is_labeled_for_interaction():
if not isinstance(self.debug, bool):
self.debug(msg="submitter_first_warning")
self.issue.add_desired_comment(
boilerplate="submitter_first_warning"
)
break
if "maintainer_review" not in labels:
if not isinstance(self.debug, bool):
self.debug(msg="maintainer_first_warning")
self.issue.add_desired_comment(
boilerplate="maintainer_first_warning"
)
break
# pending in comment.body
else:
if self.issue.is_labeled_for_interaction():
if not isinstance(self.debug, bool):
self.debug(msg="submitter_second_warning")
self.issue.add_desired_comment(
boilerplate="submitter_second_warning"
)
break
if "maintainer_review" in labels:
if not isinstance(self.debug, bool):
self.debug(msg="maintainer_second_warning")
self.issue.add_desired_comment(
boilerplate="maintainer_second_warning"
)
break
if not isinstance(self.debug, bool):
self.debug(msg="STATUS: no useful state change since last pass"
"( %s )" % comment.user.login)
break
if comment.user.login in module_maintainers \
or comment.user.login.lower() in module_maintainers\
or ('ansible' in module_maintainers and comment.user.login in self.ansible_members):
if not isinstance(self.debug, bool):
self.debug(msg="%s is module maintainer commented on %s." % (comment.user.login, comment.created_at))
if 'needs_info' in comment.body:
if not isinstance(self.debug, bool):
self.debug(msg="...said needs_info!")
self.issue.add_desired_label(name="needs_info")
elif "close_me" in comment.body:
if not isinstance(self.debug, bool): self.debug(msg="...said close_me!")
self.issue.add_desired_label(name="pending_action_close_me")
break
if comment.user.login == self.issue.get_submitter():
if not isinstance(self.debug, bool): self.debug(msg="submitter %s, commented on %s." % (comment.user.login, comment.created_at))
if comment.user.login not in self.BOTLIST and comment.user.login in self.ansible_members:
if not isinstance(self.debug, bool): self.debug(msg="%s is a ansible member" % comment.user.login)
if not isinstance(self.debug, bool): self.debug(msg="--- END Processing Comments")
def issue_type_to_label(self, issue_type):
if issue_type:
issue_type = issue_type.lower()
issue_type = issue_type.replace(' ', '_')
issue_type = issue_type.replace('documentation', 'docs')
return issue_type
def check_safe_match(self):
""" Turn force on or off depending on match characteristics """
safe_match = False
if self.action_count() == 0:
safe_match = True
elif not self.actions['close'] and not self.actions['unlabel']:
if len(self.actions['newlabel']) == 1:
if self.actions['newlabel'][0].startswith('affects_'):
safe_match = True
else:
safe_match = False
if self.module:
if self.module in self.issue.instance.title.lower():
safe_match = True
# be more lenient on re-notifications
if not safe_match:
if not self.actions['close'] and \
not self.actions['unlabel'] and \
not self.actions['newlabel']:
if len(self.actions['comments']) == 1:
if 'still waiting' in self.actions['comments'][0]:
safe_match = True
#import epdb; epdb.st()
if safe_match:
self.force = True
else:
self.force = False
def action_count(self):
""" Return the number of actions that are to be performed """
count = 0
for k,v in self.actions.iteritems():
if k == 'close' and v:
count += 1
elif k != 'close':
count += len(v)
return count
def apply_actions(self):
action_meta = {'REDO': False}
if self.safe_force:
self.check_safe_match()
if self.action_count() > 0:
if self.dry_run:
print("Dry-run specified, skipping execution of actions")
else:
if self.force:
print("Running actions non-interactive as you forced.")
self.execute_actions()
return
cont = raw_input("Take recommended actions (y/N/a/R/T)? ")
if cont in ('a', 'A'):
sys.exit(0)
if cont in ('Y', 'y'):
self.execute_actions()
if cont == 'T':
self.template_wizard()
action_meta['REDO'] = True
if cont == 'r' or cont == 'R':
action_meta['REDO'] = True
if cont == 'DEBUG':
import epdb; epdb.st()
elif self.always_pause:
print("Skipping, but pause.")
cont = raw_input("Continue (Y/n/a/R/T)? ")
if cont in ('a', 'A', 'n', 'N'):
sys.exit(0)
if cont == 'T':
self.template_wizard()
action_meta['REDO'] = True
elif cont == 'REDO':
action_meta['REDO'] = True
elif cont == 'DEBUG':
import epdb; epdb.st()
action_meta['REDO'] = True
else:
print("Skipping.")
# let the upper level code redo this issue
return action_meta
def template_wizard(self):
print('################################################')
print(self.issue.new_description)
print('################################################')
cont = raw_input("Apply this new description? (Y/N)")
if cont == 'Y':
self.issue.set_description(self.issue.new_description)
#import epdb; epdb.st()
def execute_actions(self):
"""Turns the actions into API calls"""
#time.sleep(1)
for comment in self.actions['comments']:
#import epdb; epdb.st()
if not isinstance(self.debug, bool):
self.debug(msg="API Call comment: " + comment)
self.issue.add_comment(comment=comment)
if self.actions['close']:
# https://github.com/PyGithub/PyGithub/blob/master/github/Issue.py#L263
self.issue.instance.edit(state='closed')
return
for unlabel in self.actions['unlabel']:
if not isinstance(self.debug, bool):
self.debug(msg="API Call unlabel: " + unlabel)
self.issue.remove_label(label=unlabel)
for newlabel in self.actions['newlabel']:
if not isinstance(self.debug, bool):
self.debug(msg="API Call newlabel: " + newlabel)
self.issue.add_label(label=newlabel)
if 'assign' in self.actions:
for user in self.actions['assign']:
self.issue.assign_user(user)
if 'unassign' in self.actions:
for user in self.actions['unassign']:
self.issue.unassign_user(user)
def smart_match_module(self):
'''Fuzzy matching for modules'''
if hasattr(self, 'meta'):
self.meta['smart_match_module_called'] = True
match = None
known_modules = []
for k,v in self.module_indexer.modules.iteritems():
known_modules.append(v['name'])
title = self.issue.instance.title.lower()
title = title.replace(':', '')
title_matches = [x for x in known_modules if x + ' module' in title]
if not title_matches:
title_matches = [x for x in known_modules if title.startswith(x + ' ')]
if not title_matches:
title_matches = [x for x in known_modules if ' ' + x + ' ' in title]
cmatches = None
if self.template_data.get('component name'):
component = self.template_data.get('component name')
cmatches = [x for x in known_modules if x in component]
cmatches = [x for x in cmatches if not '_' + x in component]
import epdb; epdb.st()
# use title ... ?
if title_matches:
cmatches = [x for x in cmatches if x in title_matches]
if cmatches:
if len(cmatches) >= 1:
match = cmatches[0]
if not match:
if 'docs.ansible.com' in component:
#import epdb; epdb.st()
pass
else:
#import epdb; epdb.st()
pass
#import epdb; epdb.st()
if not match:
if len(title_matches) == 1:
match = title_matches[0]
else:
print("module - title matches: %s" % title_matches)
print("module - component matches: %s" % cmatches)
return match
def cache_issue(self, issue):
iid = issue.instance.number
fpath = os.path.join(self.cachedir, str(iid))
if not os.path.isdir(fpath):
os.makedirs(fpath)
fpath = os.path.join(fpath, 'iwrapper.pickle')
with open(fpath, 'wb') as f:
pickle.dump(issue, f)
#import epdb; epdb.st()
def load_cached_issues(self, state='open'):
issues = []
idirs = glob.glob('%s/*' % self.cachedir)
idirs = [x for x in idirs if not x.endswith('.pickle')]
for idir in idirs:
wfile = os.path.join(idir, 'iwrapper.pickle')
if os.path.isfile(wfile):
with open(wfile, 'rb') as f:
wrapper = pickle.load(f)
issues.append(wrapper.instance)
return issues
def wait_for_rate_limit(self):
gh = self._connect()
GithubWrapper.wait_for_rate_limit(githubobj=gh)
@ratecheck()
def is_pr_merged(self, number):
'''Check if a PR# has been merged or not'''
merged = False
pr = None
try:
pr = self.repo.get_pullrequest(number)
except Exception as e:
print(e)
if pr:
merged = pr.merged
return merged
def print_comment_list(self):
"""Print comment creators and the commands they used"""
for x in self.issue.current_comments:
command = None
if x.user.login != 'ansibot':
command = [y for y in self.VALID_COMMANDS if y in x.body \
and not '!' + y in x.body]
command = ', '.join(command)
else:
# What template did ansibot use?
try:
command = x.body.split('\n')[-1].split()[-2]
except:
pass
if command:
print("\t%s %s (%s)" % (x.created_at.isoformat(),
x.user.login, command))
else:
print("\t%s %s" % (x.created_at.isoformat(), x.user.login))
class DefaultTriager(object):
BOTLIST = ['gregdek', 'robynbergeron', 'ansibot']
VALID_ISSUE_TYPES = ['bug report', 'feature idea', 'documentation report']
IGNORE_LABELS = [
"aws",
"azure",
"cloud",
"feature_pull_request",
"feature_idea",
"bugfix_pull_request",
"bug_report",
"docs_pull_request",
"docs_report",
"in progress",
"docs_pull_request",
"easyfix",
"pending_action",
"gce",
"python3",
"P1",
"P2",
"P3",
"P4",
]
def __init__(self,
verbose=None,
github_user=None,
github_pass=None,
github_token=None,
github_repo=None,
number=None,
start_at=None,
always_pause=False,
force=False,
safe_force=False,
dry_run=False,
no_since=False):
self.verbose = verbose
self.github_user = github_user
self.github_pass = github_pass
self.github_token = github_token
self.github_repo = github_repo
self.number = number
self.start_at = start_at
self.always_pause = always_pause
self.force = force
self.safe_force = safe_force
self.dry_run = dry_run
self.no_since = no_since
self.issue = None
self.maintainers = {}
self.module_maintainers = []
self.actions = {
'newlabel': [],
'unlabel': [],
'comments': [],
'close': False,
}
# set the cache dir
self.cachedir = '~/.ansibullbot/cache'
if self.github_repo == 'ansible':
self.cachedir += '/ansible/ansible/'
else:
self.cachedir += '/ansible/ansible-modules-%s/' % self.github_repo
self.cachedir += 'issues'
self.cachedir = os.path.expanduser(self.cachedir)
if not os.path.isdir(self.cachedir):
os.makedirs(self.cachedir)
print("Initializing AnsibleVersionIndexer")
self.version_indexer = AnsibleVersionIndexer()
#import epdb; epdb.st()
print("Initializing ModuleIndexer")
self.module_indexer = ModuleIndexer()
self.module_indexer.get_ansible_modules()
print("Initializing FileIndexer")
self.file_indexer = FileIndexer()
self.file_indexer.get_files()
print("Getting ansible members")
self.ansible_members = self.get_ansible_members()
print("Getting valid labels")
self.valid_labels = self.get_valid_labels()
# processed metadata
self.meta = {}
def _process(self, usecache=True):
'''Do some initial processing of the issue'''
# clear all actions
self.actions = {
'newlabel': [],
'unlabel': [],
'comments': [],
'close': False,
}
# clear module maintainers
self.module_maintainers = []
# print some general info about the Issue to be processed
print("\n")
print("Issue #%s [%s]: %s" %
(self.issue.number, self.icount,
self.issue.instance.title.encode('ascii', 'ignore')))
print("%s" % self.issue.instance.html_url)
print("Created at %s" % self.issue.instance.created_at)
print("Updated at %s" % self.issue.instance.updated_at)
# get the template data
self.template_data = self.issue.get_template_data()
# was the issue type defined correctly?
issue_type_defined = False
issue_type_valid = False
issue_type = False
if 'issue type' in self.template_data:
issue_type_defined = True
issue_type = self.template_data['issue type']
if issue_type.lower() in self.VALID_ISSUE_TYPES:
issue_type_valid = True
self.meta['issue_type_defined'] = issue_type_defined
self.meta['issue_type_valid'] = issue_type_valid
self.meta['issue_type'] = issue_type
if self.meta['issue_type_valid']:
self.meta['issue_type_label'] = self.issue_type_to_label(
issue_type)
else:
self.meta['issue_type_label'] = None
# What is the ansible version?
self.ansible_version = self.get_ansible_version()
if not isinstance(self.debug, bool):
self.debug('version: %s' % self.ansible_version)
self.ansible_label_version = self.get_ansible_version_major_minor()
if not isinstance(self.debug, bool):
self.debug('lversion: %s' % self.ansible_label_version)
# was component specified?
component_defined = 'component name' in self.template_data
self.meta['component_defined'] = component_defined
# extract the component
component = self.template_data.get('component name', None)
# save the real name
if self.github_repo != 'ansible':
self.match = self.module_indexer.find_match(component) or {}
else:
self.match = \
self.module_indexer.find_match(component, exact=True) or {}
self.module = self.match.get('name', None)
# check if component is a known module
component_isvalid = self.module_indexer.is_valid(component)
self.meta['component_valid'] = component_isvalid
# smart match modules (only on module repos)
if not component_isvalid and \
self.github_repo != 'ansible' and \
not self.match:
if hasattr(self, 'meta'):
self.meta['fuzzy_match_called'] = True
kwargs = dict(repo=self.github_repo,
title=self.issue.instance.title,
component=self.template_data.get('component name'))
smatch = self.module_indexer.fuzzy_match(**kwargs)
if self.module_indexer.is_valid(smatch):
self.module = smatch
component = smatch
self.match = self.module_indexer.find_match(smatch)
component_isvalid = self.module_indexer.is_valid(component)
self.meta['component_valid'] = component_isvalid
# Allow globs for module groups
# https://github.com/ansible/ansible-modules-core/issues/3831
craw = self.template_data.get('component_raw')
if self.module_indexer.is_multi(craw):
self.meta['multiple_components'] = True
# get all of the matches
self.matches = self.module_indexer.multi_match(craw)
if self.matches:
# get maintainers for all of the matches
mmap = {}
for match in self.matches:
key = match['filename']
mmap[key] = self.get_maintainers_by_match(match)
# is there a match that represents all included maintainers?
mtuples = [x[1] for x in mmap.items()]
umtuples = [list(x) for x in set(tuple(x) for x in mtuples)]
all_maintainers = []
for mtup in umtuples:
for x in mtup:
if x not in all_maintainers:
all_maintainers.append(x)
best_match = None
for k, v in mmap.iteritems():
if sorted(set(v)) == sorted(set(all_maintainers)):
best_match = k
break
if best_match:
self.match = self.module_indexer.find_match(best_match)
else:
# there's no good match that would include all maintainers
# just skip multi-module processing for now since the rest
# of the code doesn't know what to do with it.
if not isinstance(self.debug, bool):
self.debug('multi-match maintainers: %s' % umtuples)
#print(craw)
#import epdb; epdb.st()
pass
else:
self.meta['multiple_components'] = False
# set the maintainer(s)
self.module_maintainer = [x for x in self.get_module_maintainers()]
self.meta['module_maintainers'] = self.module_maintainer
# fixme: too many places where the module is set
if self.match:
self.module = self.match['name']
# Helper to fix issue descriptions ...
DF = DescriptionFixer(self.issue, self.module_indexer, self.match)
self.issue.new_description = DF.new_description
@RateLimited
def _connect(self):
"""Connects to GitHub's API"""
if self.github_token != 'False':
return Github(login_or_token=self.github_token)
else:
return Github(login_or_token=self.github_user,
password=self.github_pass)
def _get_repo_path(self):
if self.github_repo in ['core', 'extras']:
return "ansible/ansible-modules-%s" % self.github_repo
else:
return "ansible/%s" % self.github_repo
def is_pr(self, issue):
if '/pull/' in issue.html_url:
return True
else:
return False
def is_issue(self, issue):
return not self.is_pr(issue)
@RateLimited
def get_ansible_members(self):
ansible_members = []
update = False
write_cache = False
now = self.get_current_time()
org = self._connect().get_organization("ansible")
cachedir = self.cachedir
if cachedir.endswith('/issues'):
cachedir = os.path.dirname(cachedir)
cachefile = os.path.join(cachedir, 'members.pickle')
if not os.path.isdir(cachedir):
os.makedirs(cachedir)
if os.path.isfile(cachefile):
with open(cachefile, 'rb') as f:
mdata = pickle.load(f)
ansible_members = mdata[1]
if mdata[0] < org.updated_at:
update = True
else:
update = True
write_cache = True
if update:
members = org.get_members()
ansible_members = [x.login for x in members]
# save the data
if write_cache:
mdata = [now, ansible_members]
with open(cachefile, 'wb') as f:
pickle.dump(mdata, f)
#import epdb; epdb.st()
return ansible_members
@RateLimited
def get_ansible_core_team(self):
teamlist = [
'ansible-commit', 'ansible-community', 'ansible-commit-external'
]
teams = []
ansible_members = []
conn = self._connect()
org = conn.get_organization('ansible')
for x in org.get_teams():
if x.name in teamlist:
teams.append(x)
for x in teams:
for y in x.get_members():
ansible_members.append(y.login)
ansible_members = sorted(set(ansible_members))
return ansible_members
#@RateLimited
def get_valid_labels(self, repo=None):
# use the repo wrapper to enable caching+updating
if not self.ghw:
self.gh = self._connect()
self.ghw = GithubWrapper(self.gh)
if not repo:
# OLD workflow
self.repo = self.ghw.get_repo(self._get_repo_path())
vlabels = []
for vl in self.repo.get_labels():
vlabels.append(vl.name)
else:
# v3 workflow
rw = self.ghw.get_repo(repo)
vlabels = []
for vl in rw.get_labels():
vlabels.append(vl.name)
return vlabels
def _get_maintainers(self, usecache=True):
"""Reads all known maintainers from files and their owner namespace"""
if not self.maintainers or not usecache:
for repo in ['core', 'extras']:
f = open(MAINTAINERS_FILES[repo])
for line in f:
owner_space = (line.split(': ')[0]).strip()
maintainers_string = (line.split(': ')[-1]).strip()
self.maintainers[owner_space] = \
maintainers_string.split(' ')
f.close()
# meta is special
self.maintainers['meta'] = ['ansible']
return self.maintainers
def debug(self, msg=""):
"""Prints debug message if verbosity is given"""
if self.verbose:
print("Debug: " + msg)
def get_ansible_version(self):
aversion = None
rawdata = self.template_data.get('ansible version', '')
if rawdata:
aversion = self.version_indexer.strip_ansible_version(rawdata)
if not aversion or aversion == 'devel':
aversion = \
self.version_indexer.ansible_version_by_date(
self.issue.instance.created_at
)
if aversion:
if aversion.endswith('.'):
aversion += '0'
# re-run for versions ending with .x
if aversion:
if aversion.endswith('.x'):
aversion = self.version_indexer.strip_ansible_version(aversion)
#import epdb; epdb.st()
if self.version_indexer.is_valid_version(aversion) and \
aversion is not None:
return aversion
else:
# try to go through the submitter's comments and look for the
# first one that specifies a valid version
cversion = None
for comment in self.issue.current_comments:
if comment.user.login != self.issue.instance.user.login:
continue
xver = self.version_indexer.strip_ansible_version(comment.body)
if self.version_indexer.is_valid_version(xver):
cversion = xver
break
# use the comment version
aversion = cversion
return aversion
def get_ansible_version_by_issue(self, issuewrapper):
iw = issuewrapper
aversion = None
rawdata = iw.get_template_data().get('ansible version', '')
if rawdata:
aversion = self.version_indexer.strip_ansible_version(rawdata)
if not aversion or aversion == 'devel':
aversion = self.version_indexer.ansible_version_by_date(
self.issue.instance.created_at)
if aversion:
if aversion.endswith('.'):
aversion += '0'
# re-run for versions ending with .x
if aversion:
if aversion.endswith('.x'):
aversion = self.version_indexer.strip_ansible_version(aversion)
#import epdb; epdb.st()
if self.version_indexer.is_valid_version(aversion) and \
aversion is not None:
return aversion
else:
# try to go through the submitter's comments and look for the
# first one that specifies a valid version
cversion = None
for comment in self.issue.current_comments:
if comment.user.login != self.issue.instance.user.login:
continue
xver = self.version_indexer.strip_ansible_version(comment.body)
if self.version_indexer.is_valid_version(xver):
cversion = xver
break
# use the comment version
aversion = cversion
return aversion
def get_ansible_version_major_minor(self, version=None):
if not version:
# old workflow
if not hasattr(self, 'ansible_version'):
logging.debug('breakpoint!')
import epdb
epdb.st()
return self.version_indexer.get_major_minor(self.ansible_version)
else:
# v3 workflow
return self.version_indexer.get_major_minor(version)
def get_maintainers_by_match(self, match):
module_maintainers = []
maintainers = self._get_maintainers()
if match['name'] in maintainers:
module_maintainers = maintainers[match['name']]
elif match['repo_filename'] in maintainers:
module_maintainers = maintainers[match['repo_filename']]
elif (match['deprecated_filename']) in maintainers:
module_maintainers = maintainers[match['deprecated_filename']]
elif match['namespaced_module'] in maintainers:
module_maintainers = maintainers[match['namespaced_module']]
elif match['fulltopic'] in maintainers:
module_maintainers = maintainers[match['fulltopic']]
elif (match['topic'] + '/') in maintainers:
module_maintainers = maintainers[match['topic'] + '/']
else:
pass
# Fallback to using the module author(s)
if not module_maintainers and self.match:
if self.match['authors']:
module_maintainers = [x for x in self.match['authors']]
#import epdb; epdb.st()
return module_maintainers
def get_module_maintainers(self, expand=True, usecache=True):
"""Returns the list of maintainers for the current module"""
# expand=False ... ?
if self.module_maintainers and usecache:
return self.module_maintainers
module_maintainers = []
module = self.module
if not module:
return module_maintainers
if not self.module_indexer.is_valid(module):
return module_maintainers
if self.match:
mdata = self.match
else:
mdata = self.module_indexer.find_match(module)
if mdata['repository'] != self.github_repo:
# this was detected and handled in the process loop
pass
# get cached or non-cached maintainers list
if not expand:
maintainers = self._get_maintainers(usecache=False)
else:
maintainers = self._get_maintainers()
if mdata['name'] in maintainers:
module_maintainers = maintainers[mdata['name']]
elif mdata['repo_filename'] in maintainers:
module_maintainers = maintainers[mdata['repo_filename']]
elif (mdata['deprecated_filename']) in maintainers:
module_maintainers = maintainers[mdata['deprecated_filename']]
elif mdata['namespaced_module'] in maintainers:
module_maintainers = maintainers[mdata['namespaced_module']]
elif mdata['fulltopic'] in maintainers:
module_maintainers = maintainers[mdata['fulltopic']]
elif (mdata['topic'] + '/') in maintainers:
module_maintainers = maintainers[mdata['topic'] + '/']
else:
pass
# Fallback to using the module author(s)
if not module_maintainers and self.match:
if self.match['authors']:
module_maintainers = [x for x in self.match['authors']]
# need to set the no maintainer template or assume ansible?
if not module_maintainers and self.module and self.match:
#import epdb; epdb.st()
pass
#import epdb; epdb.st()
return module_maintainers
def get_current_labels(self):
"""Pull the list of labels on this Issue"""
if not self.current_labels:
labels = self.issue.instance.labels
for label in labels:
self.current_labels.append(label.name)
return self.current_labels
def run(self):
pass
def create_actions(self):
pass
def component_from_comments(self):
"""Extracts a component name from special comments"""
# https://github.com/ansible/ansible-modules/core/issues/2618
# comments like: [module: packaging/os/zypper.py] ... ?
component = None
for idx, x in enumerate(self.issue.current_comments):
if '[' in x.body and \
']' in x.body and \
('module' in x.body or
'component' in x.body or
'plugin' in x.body):
if x.user.login in BOTLIST:
component = x.body.split()[-1]
component = component.replace('[', '')
return component
def has_maintainer_commented(self):
"""Has the maintainer -ever- commented on the issue?"""
commented = False
if self.module_maintainers:
for comment in self.issue.current_comments:
# ignore comments from submitter
if comment.user.login == self.issue.get_submitter():
continue
# "ansible" is special ...
if 'ansible' in self.module_maintainers \
and comment.user.login in self.ansible_members:
commented = True
elif comment.user.login in self.module_maintainers:
commented = True
return commented
def is_maintainer_mentioned(self):
mentioned = False
if self.module_maintainers:
for comment in self.issue.current_comments:
# "ansible" is special ...
if 'ansible' in self.module_maintainers:
for x in self.ansible_members:
if ('@%s' % x) in comment.body:
mentioned = True
break
else:
for x in self.module_maintainers:
if ('@%s' % x) in comment.body:
mentioned = True
break
return mentioned
def get_current_time(self):
#now = datetime.now()
now = datetime.utcnow()
#now = datetime.now(pytz.timezone('US/Pacific'))
#import epdb; epdb.st()
return now
def age_of_last_maintainer_comment(self):
"""How long ago did the maintainer comment?"""
last_comment = None
if self.module_maintainers:
for idx, comment in enumerate(self.issue.current_comments):
# "ansible" is special ...
is_maintainer = False
if 'ansible' in self.module_maintainers \
and comment.user.login in self.ansible_members:
is_maintainer = True
elif comment.user.login in self.module_maintainers:
is_maintainer = True
if is_maintainer:
last_comment = comment
break
if not last_comment:
return -1
else:
now = self.get_current_time()
diff = now - last_comment.created_at
age = diff.days
return age
def is_waiting_on_maintainer(self):
"""Is the issue waiting on the maintainer to comment?"""
waiting = False
if self.module_maintainers:
if not self.issue.current_comments:
return True
creator_last_index = -1
maintainer_last_index = -1
for idx, comment in enumerate(self.issue.current_comments):
if comment.user.login == self.issue.get_submitter():
if creator_last_index == -1 or idx < creator_last_index:
creator_last_index = idx
# "ansible" is special ...
is_maintainer = False
if 'ansible' in self.module_maintainers \
and comment.user.login in self.ansible_members:
is_maintainer = True
elif comment.user.login in self.module_maintainers:
is_maintainer = True
if is_maintainer and \
(maintainer_last_index == -1 or
idx < maintainer_last_index):
maintainer_last_index = idx
if creator_last_index == -1 and maintainer_last_index == -1:
waiting = True
elif creator_last_index == -1 and maintainer_last_index > -1:
waiting = False
elif creator_last_index < maintainer_last_index:
waiting = True
return waiting
def keep_current_main_labels(self):
current_labels = self.issue.get_current_labels()
for current_label in current_labels:
if current_label in self.issue.MUTUALLY_EXCLUSIVE_LABELS:
self.issue.add_desired_label(name=current_label)
def add_desired_labels_by_issue_type(self, comments=True):
"""Adds labels by defined issue type"""
issue_type = self.template_data.get('issue type', False)
if issue_type is False:
self.issue.add_desired_label('needs_info')
return
if not issue_type.lower() in self.VALID_ISSUE_TYPES:
# special handling for PRs
if self.issue.instance.pull_request:
mel = [
x for x in self.issue.current_labels
if x in self.MUTUALLY_EXCLUSIVE_LABELS
]
if not mel:
# if only adding new files, assume it is a feature
if self.patch_contains_only_new_files():
issue_type = 'feature pull request'
else:
if not isinstance(self.debug, bool):
msg = '"%s"' % issue_type
msg += ' was not a valid issue type'
msg += ', adding "needs_info"'
self.debug(msg)
self.issue.add_desired_label('needs_info')
return
else:
if not isinstance(self.debug, bool):
msg = '"%s"' % issue_type
msg += ' was not a valid issue type'
msg += ', adding "needs_info"'
self.debug(msg)
self.issue.add_desired_label('needs_info')
return
desired_label = issue_type.replace(' ', '_')
desired_label = desired_label.lower()
desired_label = desired_label.replace('documentation', 'docs')
# FIXME - shouldn't have to do this
if desired_label == 'test_pull_request':
desired_label = 'test_pull_requests'
#import epdb; epdb.st()
# is there a mutually exclusive label already?
if desired_label in self.issue.MUTUALLY_EXCLUSIVE_LABELS:
mel = [
x for x in self.issue.MUTUALLY_EXCLUSIVE_LABELS
if x in self.issue.current_labels
]
if len(mel) > 0:
return
if desired_label not in self.issue.get_current_labels():
self.issue.add_desired_label(name=desired_label)
if len(self.issue.current_comments) == 0 and comments:
# only set this if no other comments
self.issue.add_desired_comment(boilerplate='issue_new')
def patch_contains_only_new_files(self):
'''Does the PR edit any existing files?'''
oldfiles = False
for x in self.issue.files:
if x.filename.encode('ascii', 'ignore') in self.file_indexer.files:
if not isinstance(self.debug, bool):
msg = 'old file match on'
msg += ' %s' % x.filename.encode('ascii', 'ignore')
self.debug(msg)
oldfiles = True
break
return not oldfiles
def add_desired_labels_by_ansible_version(self):
if 'ansible version' not in self.template_data:
if not isinstance(self.debug, bool):
self.debug(msg="no ansible version section")
self.issue.add_desired_label(name="needs_info")
#self.issue.add_desired_comment(
# boilerplate="issue_missing_data"
#)
return
if not self.template_data['ansible version']:
if not isinstance(self.debug, bool):
self.debug(msg="no ansible version defined")
self.issue.add_desired_label(name="needs_info")
#self.issue.add_desired_comment(
# boilerplate="issue_missing_data"
#)
return
def add_desired_labels_by_namespace(self):
"""Adds labels regarding module namespaces"""
SKIPTOPICS = ['network/basics/']
if not self.match:
return False
'''
if 'component name' in self.template_data and self.match:
if self.match['repository'] != self.github_repo:
self.issue.add_desired_comment(boilerplate='issue_wrong_repo')
'''
for key in ['topic', 'subtopic']:
# ignore networking/basics
if self.match[key] and not self.match['fulltopic'] in SKIPTOPICS:
thislabel = self.issue.TOPIC_MAP.\
get(self.match[key], self.match[key])
if thislabel in self.valid_labels:
self.issue.add_desired_label(thislabel)
def render_boilerplate(self, tvars, boilerplate=None):
template = environment.get_template('%s.j2' % boilerplate)
comment = template.render(**tvars)
return comment
def render_comment(self, boilerplate=None):
"""Renders templates into comments using the boilerplate as filename"""
maintainers = self.get_module_maintainers(expand=False)
if not maintainers:
# FIXME - why?
maintainers = ['NO_MAINTAINER_FOUND']
submitter = self.issue.get_submitter()
missing_sections = [
x for x in self.issue.REQUIRED_SECTIONS
if x not in self.template_data or not self.template_data.get(x)
]
if not self.match and missing_sections:
# be lenient on component name for ansible/ansible
if self.github_repo == 'ansible' and \
'component name' in missing_sections:
missing_sections.remove('component name')
#if missing_sections:
# import epdb; epdb.st()
issue_type = self.template_data.get('issue type', None)
if issue_type:
issue_type = issue_type.lower()
correct_repo = self.match.get('repository', None)
template = environment.get_template('%s.j2' % boilerplate)
component_name = self.template_data.get('component name', 'NULL'),
comment = template.render(maintainers=maintainers,
submitter=submitter,
issue_type=issue_type,
correct_repo=correct_repo,
component_name=component_name,
missing_sections=missing_sections)
return comment
def process_comments(self):
""" Processes ISSUE comments for matching criteria to add labels"""
if self.github_user not in self.BOTLIST:
self.BOTLIST.append(self.github_user)
module_maintainers = self.get_module_maintainers()
comments = self.issue.get_comments()
today = datetime.today()
if not isinstance(self.debug, bool):
self.debug(msg="--- START Processing Comments:")
for idc, comment in enumerate(comments):
if comment.user.login in self.BOTLIST:
if not isinstance(self.debug, bool):
self.debug(msg="%s is in botlist: " % comment.user.login)
time_delta = today - comment.created_at
comment_days_old = time_delta.days
if not isinstance(self.debug, bool):
msg = "Days since last bot comment: %s" % comment_days_old
self.debug(msg=msg)
if comment_days_old > 14:
labels = self.issue.desired_labels
if 'pending' not in comment.body:
if self.issue.is_labeled_for_interaction():
if not isinstance(self.debug, bool):
self.debug(msg="submitter_first_warning")
self.issue.add_desired_comment(
boilerplate="submitter_first_warning")
break
if "maintainer_review" not in labels:
if not isinstance(self.debug, bool):
self.debug(msg="maintainer_first_warning")
self.issue.add_desired_comment(
boilerplate="maintainer_first_warning")
break
# pending in comment.body
else:
if self.issue.is_labeled_for_interaction():
if not isinstance(self.debug, bool):
self.debug(msg="submitter_second_warning")
self.issue.add_desired_comment(
boilerplate="submitter_second_warning")
break
if "maintainer_review" in labels:
if not isinstance(self.debug, bool):
self.debug(msg="maintainer_second_warning")
self.issue.add_desired_comment(
boilerplate="maintainer_second_warning")
break
if not isinstance(self.debug, bool):
msg = "STATUS: no useful state change since last pass"
msg += "( %s )" % comment.user.login
self.debug(msg=msg)
break
if comment.user.login in module_maintainers \
or comment.user.login.lower() in module_maintainers\
or ('ansible' in module_maintainers and
comment.user.login in self.ansible_members):
if not isinstance(self.debug, bool):
msg = "%s" % comment.user.login
msg = " is module maintainer commented on"
msg += "%s." % comment.created_at
self.debug(msg=msg)
if 'needs_info' in comment.body:
if not isinstance(self.debug, bool):
self.debug(msg="...said needs_info!")
self.issue.add_desired_label(name="needs_info")
elif "close_me" in comment.body:
if not isinstance(self.debug, bool):
self.debug(msg="...said close_me!")
self.issue.add_desired_label(
name="pending_action_close_me")
break
if comment.user.login == self.issue.get_submitter():
if not isinstance(self.debug, bool):
msg = "submitter %s" % comment.user.login
msg += ", commented on %s." % comment.created_at
self.debug(msg=msg)
if comment.user.login not in self.BOTLIST and \
comment.user.login in self.ansible_members:
if not isinstance(self.debug, bool):
self.debug(msg="%s is a ansible member" %
comment.user.login)
if not isinstance(self.debug, bool):
self.debug(msg="--- END Processing Comments")
def issue_type_to_label(self, issue_type):
if issue_type:
issue_type = issue_type.lower()
issue_type = issue_type.replace(' ', '_')
issue_type = issue_type.replace('documentation', 'docs')
return issue_type
def check_safe_match(self):
""" Turn force on or off depending on match characteristics """
import epdb
epdb.st()
safe_match = False
if self.action_count() == 0:
safe_match = True
elif not self.actions['close'] and not self.actions['unlabel']:
if len(self.actions['newlabel']) == 1:
if self.actions['newlabel'][0].startswith('affects_'):
safe_match = True
else:
safe_match = False
if self.module:
if self.module in self.issue.instance.title.lower():
safe_match = True
# be more lenient on re-notifications
if not safe_match:
if not self.actions['close'] and \
not self.actions['unlabel'] and \
not self.actions['newlabel']:
if len(self.actions['comments']) == 1:
if 'still waiting' in self.actions['comments'][0]:
safe_match = True
#import epdb; epdb.st()
if safe_match:
self.force = True
else:
self.force = False
def action_count(self):
""" Return the number of actions that are to be performed """
count = 0
for k, v in self.actions.iteritems():
if k in ['close', 'open', 'merge', 'close_migrated'] and v:
count += 1
elif k != 'close' and k != 'open' and \
k != 'merge' and k != 'close_migrated':
count += len(v)
return count
def apply_actions(self):
action_meta = {'REDO': False}
if self.safe_force:
self.check_safe_match()
if self.action_count() > 0:
if self.dry_run:
print("Dry-run specified, skipping execution of actions")
else:
if self.force:
print("Running actions non-interactive as you forced.")
self.execute_actions()
return action_meta
cont = raw_input(
"Take recommended actions (y/N/a/R/T/DEBUG)? ")
if cont in ('a', 'A'):
sys.exit(0)
if cont in ('Y', 'y'):
self.execute_actions()
if cont == 'T':
self.template_wizard()
action_meta['REDO'] = True
if cont == 'r' or cont == 'R':
action_meta['REDO'] = True
if cont == 'DEBUG':
# put the user into a breakpoint to do live debug
action_meta['REDO'] = True
import epdb
epdb.st()
elif self.always_pause:
print("Skipping, but pause.")
cont = raw_input("Continue (Y/n/a/R/T/DEBUG)? ")
if cont in ('a', 'A', 'n', 'N'):
sys.exit(0)
if cont == 'T':
self.template_wizard()
action_meta['REDO'] = True
elif cont == 'REDO':
action_meta['REDO'] = True
elif cont == 'DEBUG':
# put the user into a breakpoint to do live debug
import epdb
epdb.st()
action_meta['REDO'] = True
else:
print("Skipping.")
# let the upper level code redo this issue
return action_meta
def template_wizard(self):
DF = DescriptionFixer(self.issue, self.meta)
print('################################################')
print(DF.new_description)
print('################################################')
cont = raw_input("Apply this new description? (Y/N)")
if cont == 'Y':
self.issue.set_description(DF.new_description)
def execute_actions(self):
"""Turns the actions into API calls"""
#time.sleep(1)
for comment in self.actions['comments']:
logging.info("acton: comment - " + comment)
self.issue.add_comment(comment=comment)
if self.actions['close']:
# https://github.com/PyGithub/PyGithub/blob/master/github/Issue.py#L263
logging.info('action: close')
self.issue.instance.edit(state='closed')
return
if self.actions['close_migrated']:
mi = self.get_issue_by_repopath_and_number(
self.meta['migrated_issue_repo_path'],
self.meta['migrated_issue_number'])
logging.info('close migrated: %s' % mi.html_url)
mi.instance.edit(state='closed')
#import epdb; epdb.st()
for unlabel in self.actions['unlabel']:
logging.info('action: unlabel - ' + unlabel)
self.issue.remove_label(label=unlabel)
for newlabel in self.actions['newlabel']:
logging.info('action: label - ' + newlabel)
self.issue.add_label(label=newlabel)
if 'assign' in self.actions:
for user in self.actions['assign']:
logging.info('action: assign - ' + user)
self.issue.assign_user(user)
if 'unassign' in self.actions:
for user in self.actions['unassign']:
logging.info('action: unassign - ' + user)
self.issue.unassign_user(user)
if 'merge' in self.actions:
if self.actions['merge']:
self.issue.merge()
def smart_match_module(self):
'''Fuzzy matching for modules'''
if hasattr(self, 'meta'):
self.meta['smart_match_module_called'] = True
match = None
known_modules = []
for k, v in self.module_indexer.modules.iteritems():
known_modules.append(v['name'])
title = self.issue.instance.title.lower()
title = title.replace(':', '')
title_matches = [x for x in known_modules if x + ' module' in title]
if not title_matches:
title_matches = [
x for x in known_modules if title.startswith(x + ' ')
]
if not title_matches:
title_matches = [
x for x in known_modules if ' ' + x + ' ' in title
]
cmatches = None
if self.template_data.get('component name'):
component = self.template_data.get('component name')
cmatches = [x for x in known_modules if x in component]
cmatches = [x for x in cmatches if not '_' + x in component]
# use title ... ?
if title_matches:
cmatches = [x for x in cmatches if x in title_matches]
if cmatches:
if len(cmatches) >= 1:
match = cmatches[0]
if not match:
if 'docs.ansible.com' in component:
pass
else:
pass
if not match:
if len(title_matches) == 1:
match = title_matches[0]
else:
print("module - title matches: %s" % title_matches)
print("module - component matches: %s" % cmatches)
return match
def cache_issue(self, issue):
iid = issue.instance.number
fpath = os.path.join(self.cachedir, str(iid))
if not os.path.isdir(fpath):
os.makedirs(fpath)
fpath = os.path.join(fpath, 'iwrapper.pickle')
with open(fpath, 'wb') as f:
pickle.dump(issue, f)
#import epdb; epdb.st()
def load_cached_issues(self, state='open'):
issues = []
idirs = glob.glob('%s/*' % self.cachedir)
idirs = [x for x in idirs if not x.endswith('.pickle')]
for idir in idirs:
wfile = os.path.join(idir, 'iwrapper.pickle')
if os.path.isfile(wfile):
with open(wfile, 'rb') as f:
wrapper = pickle.load(f)
issues.append(wrapper.instance)
return issues
def wait_for_rate_limit(self):
gh = self._connect()
GithubWrapper.wait_for_rate_limit(githubobj=gh)
@RateLimited
def is_pr_merged(self, number, repo=None):
'''Check if a PR# has been merged or not'''
merged = False
pr = None
try:
if not repo:
pr = self.repo.get_pullrequest(number)
else:
pr = repo.get_pullrequest(number)
except Exception as e:
print(e)
if pr:
merged = pr.merged
return merged
def print_comment_list(self):
"""Print comment creators and the commands they used"""
for x in self.issue.current_comments:
command = None
if x.user.login != 'ansibot':
command = [
y for y in self.VALID_COMMANDS
if y in x.body and not '!' + y in x.body
]
command = ', '.join(command)
else:
# What template did ansibot use?
try:
command = x.body.split('\n')[-1].split()[-2]
except:
pass
if command:
print("\t%s %s (%s)" %
(x.created_at.isoformat(), x.user.login, command))
else:
print("\t%s %s" % (x.created_at.isoformat(), x.user.login))