def get_issue_summaries(self, repo_url, baseurl=None, cachefile=None):
"""Return a dict of all issue summaries with numbers as keys
Adds a compatibility method for the webscraper
Args:
repo_url (str): username/repository
baseurl (str): not used
cachefile (str): not used
"""
owner = repo_url.split(u'/', 1)[0]
repo = repo_url.split(u'/', 1)[1]
summaries = self.get_all_summaries(owner, repo)
issues = {}
for x in summaries:
issues[to_text(x[u'number'])] = x
# keep the summaries for out of band analysis
repodata = {
u'user': repo_url.split(u'/', 1)[0],
u'repo': repo_url.split(u'/', 1)[1],
}
post_to_receiver(u'summaries', repodata, issues)
return issues
def get_issue_summaries(self, repo_url, baseurl=None, cachefile=None):
"""Return a dict of all issue summaries with numbers as keys
Adds a compatibility method for the webscraper
Args:
repo_url (str): username/repository
baseurl (str): not used
cachefile (str): not used
"""
owner = repo_url.split(u'/', 1)[0]
repo = repo_url.split(u'/', 1)[1]
summaries = self.get_all_summaries(owner, repo)
issues = {}
for x in summaries:
issues[to_text(x[u'number'])] = x
# keep the summaries for out of band analysis
repodata = {
u'user': repo_url.split(u'/', 1)[0],
u'repo': repo_url.split(u'/', 1)[1],
}
post_to_receiver(u'summaries', repodata, issues)
return issues
def get_issue_summaries(self, repo_url, baseurl=None, cachefile=None):
'''Paginate through github's web interface and scrape summaries'''
# repo_url - https://github.com/ansible/ansible for example
# baseurl - an entrypoint for one-off utils to scrape specific issue
# query urls. NOTE: this disables writing a cache
# get cached
if not baseurl:
issues = self.load_summaries(repo_url)
else:
issues = {}
if not baseurl:
url = repo_url
url += '/issues'
url += '?'
url += 'q='
url += urllib2.quote('sort:updated-desc')
else:
url = baseurl
namespace = repo_url.split('/')[-2]
reponame = repo_url.split('/')[-1]
rr = self._request_url(url)
soup = BeautifulSoup(rr.text, 'html.parser')
data = self._parse_issue_summary_page(soup)
if data['issues']:
# send to receiver
post_to_receiver('html_summaries', {
'user': namespace,
'repo': reponame
}, data['issues'])
# update master list
issues.update(data['issues'])
if not baseurl:
self.dump_summaries_tmp(repo_url, issues)
while data['next_page']:
rr = self._request_url(self.baseurl + data['next_page'])
soup = BeautifulSoup(rr.text, 'html.parser')
data = self._parse_issue_summary_page(soup)
# send to receiver
post_to_receiver('html_summaries', {
'user': namespace,
'repo': reponame
}, data['issues'])
if not data['next_page'] or not data['issues']:
break
changed = []
changes = False
for k, v in data['issues'].iteritems():
if not isinstance(k, unicode):
k = u'%s' % k
if k not in issues:
changed.append(k)
changes = True
elif v != issues[k]:
changed.append(k)
changes = True
issues[k] = v
if changed:
logging.info('changed: %s' % ','.join(x for x in changed))
if not baseurl:
self.dump_summaries_tmp(repo_url, issues)
if not changes:
break
# get missing
if not baseurl:
numbers = sorted([int(x) for x in issues.keys()])
missing = [x for x in xrange(1, numbers[-1]) if x not in numbers]
for x in missing:
summary = self.get_single_issue_summary(repo_url,
x,
force=True)
if summary:
post_to_receiver('html_summaries', {
'user': namespace,
'repo': reponame
}, {x: summary})
if not isinstance(x, unicode):
x = u'%s' % x
issues[x] = summary
# get missing timestamps
if not baseurl:
numbers = sorted([int(x) for x in issues.keys()])
missing = [
x for x in numbers
if str(x) not in issues or not issues[str(x)]['updated_at']
]
for x in missing:
summary = self.get_single_issue_summary(repo_url,
x,
force=True)
if summary:
post_to_receiver('html_summaries', {
'user': namespace,
'repo': reponame
}, {x: summary})
if not isinstance(x, unicode):
x = u'%s' % x
issues[x] = summary
# save the cache
if not baseurl:
self.dump_summaries(repo_url, issues)
return issues
def get_issue_summaries(self, repo_url, baseurl=None, cachefile=None):
'''Paginate through github's web interface and scrape summaries'''
# repo_url - https://github.com/ansible/ansible for example
# baseurl - an entrypoint for one-off utils to scrape specific issue
# query urls. NOTE: this disables writing a cache
# get cached
if not baseurl:
issues = self.load_summaries(repo_url)
else:
issues = {}
if not baseurl:
url = repo_url
url += u'/issues'
url += u'?'
url += u'q='
url += urllib2.quote(u'sort:updated-desc')
else:
url = baseurl
namespace = repo_url.split(u'/')[-2]
reponame = repo_url.split(u'/')[-1]
rr = self._request_url(url)
soup = BeautifulSoup(rr.text, u'html.parser')
data = self._parse_issue_summary_page(soup)
if data[u'issues']:
# send to receiver
post_to_receiver(u'html_summaries', {u'user': namespace, u'repo': reponame}, data[u'issues'])
# update master list
issues.update(data[u'issues'])
if not baseurl:
self.dump_summaries_tmp(repo_url, issues)
while data[u'next_page']:
rr = self._request_url(self.baseurl + data[u'next_page'])
soup = BeautifulSoup(rr.text, u'html.parser')
data = self._parse_issue_summary_page(soup)
# send to receiver
post_to_receiver(u'html_summaries', {u'user': namespace, u'repo': reponame}, data[u'issues'])
if not data[u'next_page'] or not data[u'issues']:
break
changed = []
changes = False
for k, v in six.iteritems(data[u'issues']):
if not isinstance(k, unicode):
k = u'%s' % k
if k not in issues:
changed.append(k)
changes = True
elif v != issues[k]:
changed.append(k)
changes = True
issues[k] = v
if changed:
logging.info(u'changed: %s' % u','.join(x for x in changed))
if not baseurl:
self.dump_summaries_tmp(repo_url, issues)
if not changes:
break
# get missing
if not baseurl:
numbers = sorted([int(x) for x in issues.keys()])
missing = [x for x in xrange(1, numbers[-1]) if x not in numbers]
for x in missing:
summary = self.get_single_issue_summary(repo_url, x, force=True)
if summary:
post_to_receiver(u'html_summaries', {u'user': namespace, u'repo': reponame}, {x: summary})
if not isinstance(x, unicode):
x = u'%s' % x
issues[x] = summary
# get missing timestamps
if not baseurl:
numbers = sorted([int(x) for x in issues.keys()])
missing = [x for x in numbers if to_text(x) not in issues or not issues[to_text(x)][u'updated_at']]
for x in missing:
summary = self.get_single_issue_summary(repo_url, x, force=True)
if summary:
post_to_receiver(u'html_summaries', {u'user': namespace, u'repo': reponame}, {x: summary})
if not isinstance(x, unicode):
x = u'%s' % x
issues[x] = summary
# save the cache
if not baseurl:
self.dump_summaries(repo_url, issues)
return issues