def get_template_data(self): """Extract templated data from an issue body""" if self.is_issue(): tfile = '.github/ISSUE_TEMPLATE.md' else: tfile = '.github/PULL_REQUEST_TEMPLATE.md' # use the fileindexer whenever possible to conserve ratelimits if self.file_indexer: tf_content = self.file_indexer.get_file_content(tfile) else: tf = self.repo.get_file_contents(tfile) tf_content = tf.decoded_content tf_sections = extract_template_sections(tf_content) self._required_template_sections = \ [x.lower() for x in tf_sections.keys() if tf_sections[x]['required']] template_data = \ extract_template_data( self.instance.body, issue_number=self.number, issue_class=self.github_type, SECTIONS=tf_sections.keys() ) return template_data
def get_template_data(self): """Extract templated data from an issue body""" if self.is_issue(): tfile = '.github/ISSUE_TEMPLATE.md' else: tfile = '.github/PULL_REQUEST_TEMPLATE.md' # use the fileindexer whenever possible to conserve ratelimits if self.file_indexer: tf_content = self.file_indexer.get_file_content(tfile) else: try: tf = self.repo.get_file_contents(tfile) tf_content = tf.decoded_content except Exception as e: logging.warning('repo does not have {}'.format(tfile)) tf_content = '' # pull out the section names from the tempalte tf_sections = extract_template_sections(tf_content, header=self.TEMPLATE_HEADER) # what is required? self._required_template_sections = \ [x.lower() for x in tf_sections.keys() if tf_sections[x]['required']] # extract ... template_data = \ extract_template_data( self.instance.body, issue_number=self.number, issue_class=self.github_type, sections=tf_sections.keys() ) return template_data
def get_template_data(self): """Extract templated data from an issue body""" if self.is_issue(): tfile = '.github/ISSUE_TEMPLATE.md' else: tfile = '.github/PULL_REQUEST_TEMPLATE.md' # use the fileindexer whenever possible to conserve ratelimits if self.file_indexer: tf_content = self.file_indexer.get_file_content(tfile) else: try: tf = self.repo.get_file_contents(tfile) tf_content = tf.decoded_content except Exception: logging.warning('repo does not have {}'.format(tfile)) tf_content = '' # pull out the section names from the tempalte tf_sections = extract_template_sections(tf_content, header=self.TEMPLATE_HEADER) # what is required? self._required_template_sections = \ [x.lower() for x in tf_sections.keys() if tf_sections[x]['required']] # extract ... template_data = \ extract_template_data( self.instance.body, issue_number=self.number, issue_class=self.github_type, sections=tf_sections.keys() ) # try comments if the description was insufficient if len(template_data.keys()) <= 2: s_comments = self.history.get_user_comments(self.submitter) for s_comment in s_comments: _template_data = extract_template_data( s_comment, issue_number=self.number, issue_class=self.github_type, sections=tf_sections.keys()) if _template_data: for k, v in _template_data.items(): if not v: continue if v and (k not in template_data or not template_data.get(k)): template_data[k] = v if 'ANSIBLE VERSION' in tf_sections and 'ansible version' not in template_data: # FIXME - abstract this into a historywrapper method vlabels = [ x for x in self.history.history if x['event'] == 'labeled' ] vlabels = [ x for x in vlabels if x['actor'] not in ['ansibot', 'ansibotdev'] ] vlabels = [ x['label'] for x in vlabels if x['label'].startswith('affects_') ] vlabels = [x for x in vlabels if x.startswith('affects_')] versions = [x.split('_')[1] for x in vlabels] versions = [float(x) for x in versions] if versions: version = versions[-1] template_data['ansible version'] = str(version) if 'COMPONENT NAME' in tf_sections and 'component name' not in template_data: if self.is_pullrequest(): fns = self.files if fns: template_data['component name'] = '\n'.join(fns) template_data['component_raw'] = '\n'.join(fns) else: clabels = [x for x in self.labels if x.startswith('c:')] if clabels: fns = [] for clabel in clabels: clabel = clabel.replace('c:', '') fns.append('lib/ansible/' + clabel) template_data['component name'] = '\n'.join(fns) template_data['component_raw'] = '\n'.join(fns) elif 'documentation' in template_data.get('issue type', '').lower(): template_data['component name'] = 'docs' template_data['component_raw'] = 'docs' if 'ISSUE TYPE' in tf_sections and 'issue type' not in template_data: # FIXME - turn this into a real classifier based on work done in # jctanner/pr-triage repo. itype = None while not itype: for label in self.labels: if label.startswith('bug'): itype = 'bug' break elif label.startswith('feature'): itype = 'feature' break elif label.startswith('doc'): itype = 'docs' break if itype: break if self.is_pullrequest(): fns = self.files for fn in fns: if fn.startswith('doc'): itype = 'docs' break if itype: break msgs = [self.title, self.body] if self.is_pullrequest(): msgs += [ x['message'] for x in self.history.history if x['event'] == 'committed' ] msgs = [x for x in msgs if x] msgs = [x.lower() for x in msgs] for msg in msgs: if 'fix' in msg: itype = 'bug' break if 'addresses' in msg: itype = 'bug' break if 'broke' in msg: itype = 'bug' break if 'add' in msg: itype = 'feature' break if 'should' in msg: itype = 'feature' break if 'please' in msg: itype = 'feature' break if 'feature' in msg: itype = 'feature' break # quit now break if itype and itype == 'bug' and self.is_issue(): template_data['issue type'] = 'bug report' elif itype and itype == 'bug' and not self.is_issue(): template_data['issue type'] = 'bugfix pullrequest' elif itype and itype == 'feature' and self.is_issue(): template_data['issue type'] = 'feature idea' elif itype and itype == 'feature' and not self.is_issue(): template_data['issue type'] = 'feature pullrequest' elif itype and itype == 'docs' and self.is_issue(): template_data['issue type'] = 'documentation report' elif itype and itype == 'docs' and not self.is_issue(): template_data['issue type'] = 'documenation pullrequest' return template_data
def main(): # need a file indexer to get the template FI = FileIndexer(checkoutdir='/tmp/fileindexer') FI.update() # get the expected sections tf_content = FI.get_file_content('.github/ISSUE_TEMPLATE.md') tf_sections = extract_template_sections(tf_content, header='#####') required_sections = [x.lower() for x in tf_sections.keys() if tf_sections[x]['required']] if not required_sections: required_sections = ['issue type', 'component name', 'ansible version', 'summary'] section_order = list(tf_sections.items()) section_order = sorted(section_order, key=lambda x: x[1]['index']) section_order = [x[0] for x in section_order] # all known possibilities section_names = ['PLUGIN NAME', 'ANSIBLE CONFIGURATION'] + section_order + ['ENVIRONMENT'] # get the numbers script = "#!/bin/bash\n" script += "\n" script += "URL='https://github.com/ansible/ansible/issues?utf8=%E2%9C%93&q=is%3Aopen%20label%3Aneeds_template%20author%3Aansibot'\n" script += "PYTHONPATH=$(pwd) scripts/scrape_github_issues_url $URL\n" (rc, so, se) = runscript(script) numbers = json.loads(so) numbers = sorted(set(numbers)) for idn,number in enumerate(numbers): print('{} {}|{}'.format(number,idn,len(numbers))) fixed = [] iurl = 'https://api.github.com/repos/ansible/ansible/issues/{}'.format(number) irr = requests.get(iurl, headers=get_headers()) idata = irr.json() curl = idata['comments_url'] crr = requests.get(curl, headers=get_headers()) comments = crr.json() if crr.links: print('paginated comments') nextp = [x for x in crr.links.items() if x[1]['rel'] == 'next'][0][1]['url'] while nextp: nrr = requests.get(nextp, headers=get_headers()) comments += nrr.json() try: nextp = [x for x in nrr.links.items() if x[1]['rel'] == 'next'][0][1]['url'] except: nextp = None #import epdb; epdb.st() newbody = idata['body'] # extract ts = run_template_extract(FI, newbody, number, 'issue', section_names) # cleanup if 'environment' in ts: ts['os / environment'] = ts['environment'] ts.pop('environment', None) # what is missing? missing = [x for x in required_sections if x.lower() not in ts] if not missing: print('{} nothing missing'.format(number)) continue # simple sed for this one if missing == ['component name'] and 'plugin name' in newbody.lower(): if 'PLUGIN NAME' in newbody: newbody = newbody.replace('PLUGIN NAME', 'COMPONENT NAME') if 'Plugin Name' in newbody: newbody = newbody.replace('Plugin Name', 'Component Name') if 'plugin name' in newbody: newbody = newbody.replace('plugin name', 'component name') print('{} sed/plugin name/component name'.format(number)) cr = requests.patch(iurl, headers=get_headers(), data=json.dumps({'body': newbody})) if cr.status_code != 200: print('failed to edit body {}'.format(idata['html_url'])) import epdb; epdb.st() continue if 'summary' in missing: ts['summary'] = newbody missing.remove('summary') fixed.append('summary') if 'issue type' in missing: # get migrated issue try: mi = get_migrated_issue(idata['body']) except Exception as e: print(e) mi = None if mi: itype = None # get issue type label from migrated issue mi_labels = [x['name'] for x in mi['labels']] if 'bug_report' in mi_labels: itype = 'Bug Report' elif 'feature_idea' in mi_labels: itype = 'Feature Idea' elif 'docs_report' in mi_labels: itype = 'Documentation Report' if itype is not None: ts['issue type'] = itype missing.remove('issue type') fixed.append('issue type') if 'component name' in missing: component = find_component(idata, ts, newbody, comments) if component: missing.remove('component name') ts['component name'] = component fixed.append('component name') if 'ansible version' in missing: labels = [x['name'] for x in idata['labels']] labels = [x for x in labels if x.startswith('affects_')] labels = sorted(set(labels)) if labels: version = labels[0].replace('affects_', '') else: version = "N/A" missing.remove('ansible version') ts['ansible version'] = version fixed.append('ansible version') if not missing: print('# {}'.format(idata['html_url'])) print('# title: {}'.format(idata['title'])) print('# component: {}'.format(ts['component name'])) print('# version: {}'.format(ts['ansible version'])) print('# fixed: {}'.format(fixed)) newbody = render_body(ts, section_order) print('<====================================================>') print(newbody) print('<====================================================>') import epdb; epdb.st() cr = requests.patch(iurl, headers=get_headers(), data=json.dumps({'body': newbody})) if cr.status_code != 200: print('failed to edit body {}'.format(idata['html_url'])) import epdb; epdb.st() continue print('no solution(s) for {} {}'.format(idata['html_url'], missing)) print('DONE')
def get_template_data(self): """Extract templated data from an issue body""" if self.is_issue(): tfile = u'.github/ISSUE_TEMPLATE.md' else: tfile = u'.github/PULL_REQUEST_TEMPLATE.md' # use the fileindexer whenever possible to conserve ratelimits if self.file_indexer: tf_content = self.file_indexer.get_file_content(tfile) else: try: tf = self.repo.get_file_contents(tfile) tf_content = tf.decoded_content except Exception: logging.warning(u'repo does not have {}'.format(tfile)) tf_content = u'' # pull out the section names from the tempalte tf_sections = extract_template_sections(tf_content, header=self.TEMPLATE_HEADER) # what is required? self._required_template_sections = \ [x.lower() for x in tf_sections.keys() if tf_sections[x][u'required']] # extract ... template_data = \ extract_template_data( self.instance.body, issue_number=self.number, issue_class=self.github_type, sections=tf_sections.keys() ) # try comments if the description was insufficient if len(template_data.keys()) <= 2: s_comments = self.history.get_user_comments(self.submitter) for s_comment in s_comments: _template_data = extract_template_data( s_comment, issue_number=self.number, issue_class=self.github_type, sections=tf_sections.keys() ) if _template_data: for k, v in _template_data.items(): if not v: continue if v and (k not in template_data or not template_data.get(k)): template_data[k] = v if u'ANSIBLE VERSION' in tf_sections and u'ansible version' not in template_data: # FIXME - abstract this into a historywrapper method vlabels = [x for x in self.history.history if x[u'event'] == u'labeled'] vlabels = [x for x in vlabels if x[u'actor'] not in [u'ansibot', u'ansibotdev']] vlabels = [x[u'label'] for x in vlabels if x[u'label'].startswith(u'affects_')] vlabels = [x for x in vlabels if x.startswith(u'affects_')] versions = [x.split(u'_')[1] for x in vlabels] versions = [float(x) for x in versions] if versions: version = versions[-1] template_data[u'ansible version'] = to_text(version) if u'COMPONENT NAME' in tf_sections and u'component name' not in template_data: if self.is_pullrequest(): fns = self.files if fns: template_data[u'component name'] = u'\n'.join(fns) template_data[u'component_raw'] = u'\n'.join(fns) else: clabels = [x for x in self.labels if x.startswith(u'c:')] if clabels: fns = [] for clabel in clabels: clabel = clabel.replace(u'c:', u'') fns.append(u'lib/ansible/' + clabel) template_data[u'component name'] = u'\n'.join(fns) template_data[u'component_raw'] = u'\n'.join(fns) elif u'documentation' in template_data.get(u'issue type', u'').lower(): template_data[u'component name'] = u'docs' template_data[u'component_raw'] = u'docs' if u'ISSUE TYPE' in tf_sections and u'issue type' not in template_data: # FIXME - turn this into a real classifier based on work done in # jctanner/pr-triage repo. itype = None while not itype: for label in self.labels: if label.startswith(u'bug'): itype = u'bug' break elif label.startswith(u'feature'): itype = u'feature' break elif label.startswith(u'doc'): itype = u'docs' break if itype: break if self.is_pullrequest(): fns = self.files for fn in fns: if fn.startswith(u'doc'): itype = u'docs' break if itype: break msgs = [self.title, self.body] if self.is_pullrequest(): msgs += [x[u'message'] for x in self.history.history if x[u'event'] == u'committed'] msgs = [x for x in msgs if x] msgs = [x.lower() for x in msgs] for msg in msgs: if u'fix' in msg: itype = u'bug' break if u'addresses' in msg: itype = u'bug' break if u'broke' in msg: itype = u'bug' break if u'add' in msg: itype = u'feature' break if u'should' in msg: itype = u'feature' break if u'please' in msg: itype = u'feature' break if u'feature' in msg: itype = u'feature' break # quit now break if itype and itype == u'bug' and self.is_issue(): template_data[u'issue type'] = u'bug report' elif itype and itype == u'bug' and not self.is_issue(): template_data[u'issue type'] = u'bugfix pullrequest' elif itype and itype == u'feature' and self.is_issue(): template_data[u'issue type'] = u'feature idea' elif itype and itype == u'feature' and not self.is_issue(): template_data[u'issue type'] = u'feature pullrequest' elif itype and itype == u'docs' and self.is_issue(): template_data[u'issue type'] = u'documentation report' elif itype and itype == u'docs' and not self.is_issue(): template_data[u'issue type'] = u'documenation pullrequest' return template_data