def get_template_data(self):
"""Extract templated data from an issue body"""
if self.is_issue():
tfile = '.github/ISSUE_TEMPLATE.md'
else:
tfile = '.github/PULL_REQUEST_TEMPLATE.md'
# use the fileindexer whenever possible to conserve ratelimits
if self.file_indexer:
tf_content = self.file_indexer.get_file_content(tfile)
else:
tf = self.repo.get_file_contents(tfile)
tf_content = tf.decoded_content
tf_sections = extract_template_sections(tf_content)
self._required_template_sections = \
[x.lower() for x in tf_sections.keys()
if tf_sections[x]['required']]
template_data = \
extract_template_data(
self.instance.body,
issue_number=self.number,
issue_class=self.github_type,
SECTIONS=tf_sections.keys()
)
return template_data
def get_template_data(self):
"""Extract templated data from an issue body"""
if self.is_issue():
tfile = '.github/ISSUE_TEMPLATE.md'
else:
tfile = '.github/PULL_REQUEST_TEMPLATE.md'
# use the fileindexer whenever possible to conserve ratelimits
if self.file_indexer:
tf_content = self.file_indexer.get_file_content(tfile)
else:
try:
tf = self.repo.get_file_contents(tfile)
tf_content = tf.decoded_content
except Exception as e:
logging.warning('repo does not have {}'.format(tfile))
tf_content = ''
# pull out the section names from the tempalte
tf_sections = extract_template_sections(tf_content, header=self.TEMPLATE_HEADER)
# what is required?
self._required_template_sections = \
[x.lower() for x in tf_sections.keys()
if tf_sections[x]['required']]
# extract ...
template_data = \
extract_template_data(
self.instance.body,
issue_number=self.number,
issue_class=self.github_type,
sections=tf_sections.keys()
)
return template_data
def get_template_data(self):
"""Extract templated data from an issue body"""
if self.is_issue():
tfile = '.github/ISSUE_TEMPLATE.md'
else:
tfile = '.github/PULL_REQUEST_TEMPLATE.md'
# use the fileindexer whenever possible to conserve ratelimits
if self.file_indexer:
tf_content = self.file_indexer.get_file_content(tfile)
else:
try:
tf = self.repo.get_file_contents(tfile)
tf_content = tf.decoded_content
except Exception:
logging.warning('repo does not have {}'.format(tfile))
tf_content = ''
# pull out the section names from the tempalte
tf_sections = extract_template_sections(tf_content,
header=self.TEMPLATE_HEADER)
# what is required?
self._required_template_sections = \
[x.lower() for x in tf_sections.keys()
if tf_sections[x]['required']]
# extract ...
template_data = \
extract_template_data(
self.instance.body,
issue_number=self.number,
issue_class=self.github_type,
sections=tf_sections.keys()
)
# try comments if the description was insufficient
if len(template_data.keys()) <= 2:
s_comments = self.history.get_user_comments(self.submitter)
for s_comment in s_comments:
_template_data = extract_template_data(
s_comment,
issue_number=self.number,
issue_class=self.github_type,
sections=tf_sections.keys())
if _template_data:
for k, v in _template_data.items():
if not v:
continue
if v and (k not in template_data
or not template_data.get(k)):
template_data[k] = v
if 'ANSIBLE VERSION' in tf_sections and 'ansible version' not in template_data:
# FIXME - abstract this into a historywrapper method
vlabels = [
x for x in self.history.history if x['event'] == 'labeled'
]
vlabels = [
x for x in vlabels
if x['actor'] not in ['ansibot', 'ansibotdev']
]
vlabels = [
x['label'] for x in vlabels
if x['label'].startswith('affects_')
]
vlabels = [x for x in vlabels if x.startswith('affects_')]
versions = [x.split('_')[1] for x in vlabels]
versions = [float(x) for x in versions]
if versions:
version = versions[-1]
template_data['ansible version'] = str(version)
if 'COMPONENT NAME' in tf_sections and 'component name' not in template_data:
if self.is_pullrequest():
fns = self.files
if fns:
template_data['component name'] = '\n'.join(fns)
template_data['component_raw'] = '\n'.join(fns)
else:
clabels = [x for x in self.labels if x.startswith('c:')]
if clabels:
fns = []
for clabel in clabels:
clabel = clabel.replace('c:', '')
fns.append('lib/ansible/' + clabel)
template_data['component name'] = '\n'.join(fns)
template_data['component_raw'] = '\n'.join(fns)
elif 'documentation' in template_data.get('issue type',
'').lower():
template_data['component name'] = 'docs'
template_data['component_raw'] = 'docs'
if 'ISSUE TYPE' in tf_sections and 'issue type' not in template_data:
# FIXME - turn this into a real classifier based on work done in
# jctanner/pr-triage repo.
itype = None
while not itype:
for label in self.labels:
if label.startswith('bug'):
itype = 'bug'
break
elif label.startswith('feature'):
itype = 'feature'
break
elif label.startswith('doc'):
itype = 'docs'
break
if itype:
break
if self.is_pullrequest():
fns = self.files
for fn in fns:
if fn.startswith('doc'):
itype = 'docs'
break
if itype:
break
msgs = [self.title, self.body]
if self.is_pullrequest():
msgs += [
x['message'] for x in self.history.history
if x['event'] == 'committed'
]
msgs = [x for x in msgs if x]
msgs = [x.lower() for x in msgs]
for msg in msgs:
if 'fix' in msg:
itype = 'bug'
break
if 'addresses' in msg:
itype = 'bug'
break
if 'broke' in msg:
itype = 'bug'
break
if 'add' in msg:
itype = 'feature'
break
if 'should' in msg:
itype = 'feature'
break
if 'please' in msg:
itype = 'feature'
break
if 'feature' in msg:
itype = 'feature'
break
# quit now
break
if itype and itype == 'bug' and self.is_issue():
template_data['issue type'] = 'bug report'
elif itype and itype == 'bug' and not self.is_issue():
template_data['issue type'] = 'bugfix pullrequest'
elif itype and itype == 'feature' and self.is_issue():
template_data['issue type'] = 'feature idea'
elif itype and itype == 'feature' and not self.is_issue():
template_data['issue type'] = 'feature pullrequest'
elif itype and itype == 'docs' and self.is_issue():
template_data['issue type'] = 'documentation report'
elif itype and itype == 'docs' and not self.is_issue():
template_data['issue type'] = 'documenation pullrequest'
return template_data
def main():
# need a file indexer to get the template
FI = FileIndexer(checkoutdir='/tmp/fileindexer')
FI.update()
# get the expected sections
tf_content = FI.get_file_content('.github/ISSUE_TEMPLATE.md')
tf_sections = extract_template_sections(tf_content, header='#####')
required_sections = [x.lower() for x in tf_sections.keys() if tf_sections[x]['required']]
if not required_sections:
required_sections = ['issue type', 'component name', 'ansible version', 'summary']
section_order = list(tf_sections.items())
section_order = sorted(section_order, key=lambda x: x[1]['index'])
section_order = [x[0] for x in section_order]
# all known possibilities
section_names = ['PLUGIN NAME', 'ANSIBLE CONFIGURATION'] + section_order + ['ENVIRONMENT']
# get the numbers
script = "#!/bin/bash\n"
script += "\n"
script += "URL='https://github.com/ansible/ansible/issues?utf8=%E2%9C%93&q=is%3Aopen%20label%3Aneeds_template%20author%3Aansibot'\n"
script += "PYTHONPATH=$(pwd) scripts/scrape_github_issues_url $URL\n"
(rc, so, se) = runscript(script)
numbers = json.loads(so)
numbers = sorted(set(numbers))
for idn,number in enumerate(numbers):
print('{} {}|{}'.format(number,idn,len(numbers)))
fixed = []
iurl = 'https://api.github.com/repos/ansible/ansible/issues/{}'.format(number)
irr = requests.get(iurl, headers=get_headers())
idata = irr.json()
curl = idata['comments_url']
crr = requests.get(curl, headers=get_headers())
comments = crr.json()
if crr.links:
print('paginated comments')
nextp = [x for x in crr.links.items() if x[1]['rel'] == 'next'][0][1]['url']
while nextp:
nrr = requests.get(nextp, headers=get_headers())
comments += nrr.json()
try:
nextp = [x for x in nrr.links.items() if x[1]['rel'] == 'next'][0][1]['url']
except:
nextp = None
#import epdb; epdb.st()
newbody = idata['body']
# extract
ts = run_template_extract(FI, newbody, number, 'issue', section_names)
# cleanup
if 'environment' in ts:
ts['os / environment'] = ts['environment']
ts.pop('environment', None)
# what is missing?
missing = [x for x in required_sections if x.lower() not in ts]
if not missing:
print('{} nothing missing'.format(number))
continue
# simple sed for this one
if missing == ['component name'] and 'plugin name' in newbody.lower():
if 'PLUGIN NAME' in newbody:
newbody = newbody.replace('PLUGIN NAME', 'COMPONENT NAME')
if 'Plugin Name' in newbody:
newbody = newbody.replace('Plugin Name', 'Component Name')
if 'plugin name' in newbody:
newbody = newbody.replace('plugin name', 'component name')
print('{} sed/plugin name/component name'.format(number))
cr = requests.patch(iurl, headers=get_headers(), data=json.dumps({'body': newbody}))
if cr.status_code != 200:
print('failed to edit body {}'.format(idata['html_url']))
import epdb; epdb.st()
continue
if 'summary' in missing:
ts['summary'] = newbody
missing.remove('summary')
fixed.append('summary')
if 'issue type' in missing:
# get migrated issue
try:
mi = get_migrated_issue(idata['body'])
except Exception as e:
print(e)
mi = None
if mi:
itype = None
# get issue type label from migrated issue
mi_labels = [x['name'] for x in mi['labels']]
if 'bug_report' in mi_labels:
itype = 'Bug Report'
elif 'feature_idea' in mi_labels:
itype = 'Feature Idea'
elif 'docs_report' in mi_labels:
itype = 'Documentation Report'
if itype is not None:
ts['issue type'] = itype
missing.remove('issue type')
fixed.append('issue type')
if 'component name' in missing:
component = find_component(idata, ts, newbody, comments)
if component:
missing.remove('component name')
ts['component name'] = component
fixed.append('component name')
if 'ansible version' in missing:
labels = [x['name'] for x in idata['labels']]
labels = [x for x in labels if x.startswith('affects_')]
labels = sorted(set(labels))
if labels:
version = labels[0].replace('affects_', '')
else:
version = "N/A"
missing.remove('ansible version')
ts['ansible version'] = version
fixed.append('ansible version')
if not missing:
print('# {}'.format(idata['html_url']))
print('# title: {}'.format(idata['title']))
print('# component: {}'.format(ts['component name']))
print('# version: {}'.format(ts['ansible version']))
print('# fixed: {}'.format(fixed))
newbody = render_body(ts, section_order)
print('<====================================================>')
print(newbody)
print('<====================================================>')
import epdb; epdb.st()
cr = requests.patch(iurl, headers=get_headers(), data=json.dumps({'body': newbody}))
if cr.status_code != 200:
print('failed to edit body {}'.format(idata['html_url']))
import epdb; epdb.st()
continue
print('no solution(s) for {} {}'.format(idata['html_url'], missing))
print('DONE')
def get_template_data(self):
"""Extract templated data from an issue body"""
if self.is_issue():
tfile = u'.github/ISSUE_TEMPLATE.md'
else:
tfile = u'.github/PULL_REQUEST_TEMPLATE.md'
# use the fileindexer whenever possible to conserve ratelimits
if self.file_indexer:
tf_content = self.file_indexer.get_file_content(tfile)
else:
try:
tf = self.repo.get_file_contents(tfile)
tf_content = tf.decoded_content
except Exception:
logging.warning(u'repo does not have {}'.format(tfile))
tf_content = u''
# pull out the section names from the tempalte
tf_sections = extract_template_sections(tf_content, header=self.TEMPLATE_HEADER)
# what is required?
self._required_template_sections = \
[x.lower() for x in tf_sections.keys()
if tf_sections[x][u'required']]
# extract ...
template_data = \
extract_template_data(
self.instance.body,
issue_number=self.number,
issue_class=self.github_type,
sections=tf_sections.keys()
)
# try comments if the description was insufficient
if len(template_data.keys()) <= 2:
s_comments = self.history.get_user_comments(self.submitter)
for s_comment in s_comments:
_template_data = extract_template_data(
s_comment,
issue_number=self.number,
issue_class=self.github_type,
sections=tf_sections.keys()
)
if _template_data:
for k, v in _template_data.items():
if not v:
continue
if v and (k not in template_data or not template_data.get(k)):
template_data[k] = v
if u'ANSIBLE VERSION' in tf_sections and u'ansible version' not in template_data:
# FIXME - abstract this into a historywrapper method
vlabels = [x for x in self.history.history if x[u'event'] == u'labeled']
vlabels = [x for x in vlabels if x[u'actor'] not in [u'ansibot', u'ansibotdev']]
vlabels = [x[u'label'] for x in vlabels if x[u'label'].startswith(u'affects_')]
vlabels = [x for x in vlabels if x.startswith(u'affects_')]
versions = [x.split(u'_')[1] for x in vlabels]
versions = [float(x) for x in versions]
if versions:
version = versions[-1]
template_data[u'ansible version'] = to_text(version)
if u'COMPONENT NAME' in tf_sections and u'component name' not in template_data:
if self.is_pullrequest():
fns = self.files
if fns:
template_data[u'component name'] = u'\n'.join(fns)
template_data[u'component_raw'] = u'\n'.join(fns)
else:
clabels = [x for x in self.labels if x.startswith(u'c:')]
if clabels:
fns = []
for clabel in clabels:
clabel = clabel.replace(u'c:', u'')
fns.append(u'lib/ansible/' + clabel)
template_data[u'component name'] = u'\n'.join(fns)
template_data[u'component_raw'] = u'\n'.join(fns)
elif u'documentation' in template_data.get(u'issue type', u'').lower():
template_data[u'component name'] = u'docs'
template_data[u'component_raw'] = u'docs'
if u'ISSUE TYPE' in tf_sections and u'issue type' not in template_data:
# FIXME - turn this into a real classifier based on work done in
# jctanner/pr-triage repo.
itype = None
while not itype:
for label in self.labels:
if label.startswith(u'bug'):
itype = u'bug'
break
elif label.startswith(u'feature'):
itype = u'feature'
break
elif label.startswith(u'doc'):
itype = u'docs'
break
if itype:
break
if self.is_pullrequest():
fns = self.files
for fn in fns:
if fn.startswith(u'doc'):
itype = u'docs'
break
if itype:
break
msgs = [self.title, self.body]
if self.is_pullrequest():
msgs += [x[u'message'] for x in self.history.history if x[u'event'] == u'committed']
msgs = [x for x in msgs if x]
msgs = [x.lower() for x in msgs]
for msg in msgs:
if u'fix' in msg:
itype = u'bug'
break
if u'addresses' in msg:
itype = u'bug'
break
if u'broke' in msg:
itype = u'bug'
break
if u'add' in msg:
itype = u'feature'
break
if u'should' in msg:
itype = u'feature'
break
if u'please' in msg:
itype = u'feature'
break
if u'feature' in msg:
itype = u'feature'
break
# quit now
break
if itype and itype == u'bug' and self.is_issue():
template_data[u'issue type'] = u'bug report'
elif itype and itype == u'bug' and not self.is_issue():
template_data[u'issue type'] = u'bugfix pullrequest'
elif itype and itype == u'feature' and self.is_issue():
template_data[u'issue type'] = u'feature idea'
elif itype and itype == u'feature' and not self.is_issue():
template_data[u'issue type'] = u'feature pullrequest'
elif itype and itype == u'docs' and self.is_issue():
template_data[u'issue type'] = u'documentation report'
elif itype and itype == u'docs' and not self.is_issue():
template_data[u'issue type'] = u'documenation pullrequest'
return template_data
