def search(query, stats):
""" Perform issue search for given stats instance """
log.debug("Search query: {0}".format(query))
issues = []
# Fetch data from the server in batches of MAX_RESULTS issues
for batch in range(MAX_BATCHES):
response = stats.parent.session.get(
"{0}/rest/api/latest/search?{1}".format(
stats.parent.url, urllib.parse.urlencode({
"jql": query,
"fields": "summary,comment",
"maxResults": MAX_RESULTS,
"startAt": batch * MAX_RESULTS})))
data = response.json()
if not response.ok:
try:
error = " ".join(data["errorMessages"])
except KeyError:
error = "unknown"
raise ReportError(
f"Failed to fetch jira issues for query '{query}'. "
f"The reason was '{response.reason}' "
f"and the error was '{error}'.")
log.debug("Batch {0} result: {1} fetched".format(
batch, listed(data["issues"], "issue")))
log.data(pretty(data))
issues.extend(data["issues"])
# If all issues fetched, we're done
if len(issues) >= data["total"]:
break
# Return the list of issue objects
return [Issue(issue, prefix=stats.parent.prefix) for issue in issues]
def search(query, stats):
""" Perform issue search for given stats instance """
log.debug("Search query: {0}".format(query))
issues = []
# Fetch data from the server in batches of MAX_RESULTS issues
for batch in range(MAX_BATCHES):
result = stats.parent.session.open(
"{0}/rest/api/latest/search?{1}".format(
stats.parent.url,
urllib.urlencode({
"jql": query,
"fields": "summary,comment",
"maxResults": MAX_RESULTS,
"startAt": batch * MAX_RESULTS
})))
data = json.loads(result.read())
log.debug("Batch {0} result: {1} fetched".format(
batch, listed(data["issues"], "issue")))
log.data(pretty(data))
issues.extend(data["issues"])
# If all issues fetched, we're done
if len(issues) >= data["total"]:
break
# Return the list of issue objects
return [Issue(issue, prefix=stats.parent.prefix) for issue in issues]
def search(query, stats, expand=None):
""" Perform page/comment search for given stats instance """
log.debug("Search query: {0}".format(query))
content = []
# Fetch data from the server in batches of MAX_RESULTS issues
for batch in range(MAX_BATCHES):
response = stats.parent.session.get(
"{0}/rest/api/content/search?{1}".format(
stats.parent.url,
urllib.parse.urlencode({
"cql": query,
"limit": MAX_RESULTS,
"expand": expand,
"start": batch * MAX_RESULTS
})))
data = response.json()
log.debug("Batch {0} result: {1} fetched".format(
batch, listed(data["results"], "object")))
log.data(pretty(data))
content.extend(data["results"])
# If all issues fetched, we're done
if data['_links'].get('next') is None:
break
return content
def search(self, query):
""" Perform GitHub query """
result = []
url = self.url + "/" + query + f"&per_page={PER_PAGE}"
while True:
# Fetch the query
log.debug(f"GitHub query: {url}")
try:
response = requests.get(url, headers=self.headers)
log.debug(f"Response headers:\n{response.headers}")
except requests.exceptions.RequestException as error:
log.debug(error)
raise ReportError(f"GitHub search on {self.url} failed.")
# Parse fetched json data
try:
data = json.loads(response.text)["items"]
result.extend(data)
except requests.exceptions.JSONDecodeError as error:
log.debug(error)
raise ReportError(f"GitHub JSON failed: {response.text}.")
# Update url to the next page, break if no next page provided
if 'next' in response.links:
url = response.links['next']['url']
else:
break
log.debug("Result: {0} fetched".format(listed(len(result), "item")))
log.data(pretty(result))
return result
def __init__(self, option, name=None, parent=None, user=None):
name = "Trello updates for {0}".format(option)
super(TrelloStatsGroup, self).__init__(option=option,
name=name,
parent=parent,
user=user)
# map appropriate API methods to Classes
filter_map = {
'Boards': {},
'Lists': {},
'Cards': {
'commentCard': TrelloCardsCommented,
'updateCard': TrelloCardsUpdated,
'updateCard:closed': TrelloCardsClosed,
'updateCard:idList': TrelloCardsMoved,
'createCard': TrelloCardsCreated
},
'Checklists': {
'updateCheckItemStateOnCard': TrelloCheckItem
}
}
self._session = None
self.url = "https://trello.com/1"
config = dict(Config().section(option))
positional_args = ['apikey', 'token']
if (not set(positional_args).issubset(set(config.keys()))
and "user" not in config):
raise ReportError(
"No ({0}) or 'user' set in the [{1}] section".format(
listed(positional_args, quote="'"), option))
optional_args = ["board_links", "apikey", "token"]
for arg in optional_args:
if arg not in config:
config[arg] = ""
# Skip API instance initialization when building options
if self.options is None:
trello = None
else:
trello = TrelloAPI(stats=self, config=config)
try:
filters = split(config["filters"])
except KeyError:
filters = DEFAULT_FILTERS
for filt_group in sorted(filter_map):
for filt in sorted(filter_map[filt_group]):
if filters != [""] and filt not in filters:
continue
self.stats.append(filter_map[filt_group][filt](trello=trello,
filt=filt,
option=option +
"-" + filt,
parent=self))
def test_listed():
from did.utils import listed
assert listed
assert listed(range(1)) == "0"
assert listed(range(2)) == "0 and 1"
assert listed(range(3), quote='"') == '"0", "1" and "2"'
assert listed(range(4), max=3) == "0, 1, 2 and 1 more"
assert listed(range(5), 'number', max=3) == "0, 1, 2 and 2 more numbers"
assert listed(range(6), 'category') == "6 categories"
assert listed(7, "leaf", "leaves") == "7 leaves"
assert listed([], "item", max=0) == "0 items"
def test_listed():
from did.utils import listed
assert listed
assert listed(range(1)) == "0"
assert listed(range(2)) == "0 and 1"
assert listed(range(3), quote='"') == '"0", "1" and "2"'
assert listed(range(4), max=3) == "0, 1, 2 and 1 more"
assert listed(range(5), 'number', max=3) == "0, 1, 2 and 2 more numbers"
assert listed(range(6), 'category') == "6 categories"
assert listed(7, "leaf", "leaves") == "7 leaves"
assert listed([], "item", max=0) == "0 items"
def __init__(self, option, name=None, parent=None, user=None):
name = "Trello updates for {0}".format(option)
super(TrelloStatsGroup, self).__init__(
option=option, name=name, parent=parent, user=user)
# map appropriate API methods to Classes
filter_map = {
'Boards': {},
'Lists': {},
'Cards': {
'commentCard': TrelloCardsCommented,
'updateCard': TrelloCardsUpdated,
'updateCard:closed': TrelloCardsClosed,
'updateCard:idList': TrelloCardsMoved,
'createCard': TrelloCardsCreated},
'Checklists': {
'updateCheckItemStateOnCard': TrelloCheckItem}
}
self._session = None
self.url = "https://trello.com/1"
config = dict(Config().section(option))
positional_args = ['apikey', 'token']
if (not set(positional_args).issubset(set(config.keys()))
and "user" not in config):
raise ReportError(
"No ({0}) or 'user' set in the [{1}] section".format(
listed(positional_args, quote="'"), option))
optional_args = ["board_links", "apikey", "token"]
for arg in optional_args:
if arg not in config:
config[arg] = ""
# Skip API instance initialization when building options
if self.options is None:
trello = None
else:
trello = TrelloAPI(stats=self, config=config)
try:
filters = split(config["filters"])
except KeyError:
filters = DEFAULT_FILTERS
for filt_group in filter_map:
for filt in filter_map[filt_group]:
if filters != [""] and filt not in filters:
continue
self.stats.append(filter_map[filt_group][filt](
trello=trello,
filt=filt,
option=option + "-" + filt,
parent=self))
def search(self, user, since, until, target_type, action_name):
""" Perform GitLab query """
if not self.user:
self.user = self.get_user(user)
if not self.events:
self.events = self.user_events(self.user['id'], since, until)
result = []
for event in self.events:
created_at = dateutil.parser.parse(event['created_at']).date()
if (event['target_type'] == target_type
and event['action_name'] == action_name
and since.date <= created_at and until.date >= created_at):
result.append(event)
log.debug("Result: {0} fetched".format(listed(len(result), "item")))
return result
def search(self, user, since, until, target_type, action_name):
""" Perform GitLab query """
if not self.user:
self.user = self.get_user(user)
if not self.events:
self.events = self.user_events(self.user['id'], since, until)
result = []
for event in self.events:
created_at = dateutil.parser.parse(event['created_at']).date()
if (event['target_type'] == target_type and
event['action_name'] == action_name and
since.date <= created_at and until.date >= created_at):
result.append(event)
log.debug("Result: {0} fetched".format(listed(len(result), "item")))
return result
def search(self, query):
""" Perform GitHub query """
url = self.url + "/" + query
log.debug("GitHub query: {0}".format(url))
try:
request = urllib.request.Request(url, headers=self.headers)
response = urllib.request.urlopen(request)
log.debug("Response headers:\n{0}".format(
str(response.info()).strip()))
except urllib.error.URLError as error:
log.debug(error)
raise ReportError("GitHub search on {0} failed.".format(self.url))
result = json.loads(response.read())["items"]
log.debug("Result: {0} fetched".format(listed(len(result), "item")))
log.data(pretty(result))
return result
def search(self, query):
""" Perform GitHub query """
url = self.url + "/" + query
log.debug("GitHub query: {0}".format(url))
try:
request = urllib2.Request(url, headers=self.headers)
response = urllib2.urlopen(request)
log.debug("Response headers:\n{0}".format(
unicode(response.info()).strip()))
except urllib2.URLError as error:
log.debug(error)
raise ReportError(
"GitHub search on {0} failed.".format(self.url))
result = json.loads(response.read())["items"]
log.debug("Result: {0} fetched".format(listed(len(result), "item")))
log.data(pretty(result))
return result
def search(self, query):
""" Perform Bodhi query """
result = []
current_page = 1
original_query = query
while current_page:
log.debug("Bodhi query: {0}".format(query))
client = BodhiClient(self.url)
data = client.send_request(query, verb='GET')
objects = data['updates']
log.debug("Result: {0} fetched".format(listed(
len(objects), "item")))
log.data(pretty(data))
result.extend(objects)
if current_page < data['pages']:
current_page = current_page + 1
query = f"{original_query}&page={current_page}"
else:
current_page = None
return result
def search(self, query, pagination, result_field):
""" Perform Pagure query """
result = []
url = "/".join((self.url, query))
while url:
log.debug("Pagure query: {0}".format(url))
try:
response = requests.get(url, headers=self.headers)
log.data("Response headers:\n{0}".format(response.headers))
except requests.RequestException as error:
log.error(error)
raise ReportError("Pagure search {0} failed.".format(self.url))
data = response.json()
objects = data[result_field]
log.debug("Result: {0} fetched".format(listed(
len(objects), "item")))
log.data(pretty(data))
# FIXME later: Work around https://pagure.io/pagure/issue/4057
if not objects:
break
result.extend(objects)
url = data[pagination]['next']
return result
def search(query, stats):
""" Perform issue search for given stats instance """
log.debug("Search query: {0}".format(query))
issues = []
# Fetch data from the server in batches of MAX_RESULTS issues
for batch in range(MAX_BATCHES):
result = stats.parent.session.open(
"{0}/rest/api/latest/search?{1}".format(
stats.parent.url, urllib.urlencode({
"jql": query,
"fields": "summary,comment",
"maxResults": MAX_RESULTS,
"startAt": batch * MAX_RESULTS})))
data = json.loads(result.read())
log.debug("Batch {0} result: {1} fetched".format(
batch, listed(data["issues"], "issue")))
log.data(pretty(data))
issues.extend(data["issues"])
# If all issues fetched, we're done
if len(issues) >= data["total"]:
break
# Return the list of issue objects
return [Issue(issue, prefix=stats.parent.prefix) for issue in issues]
def search(self, query, pagination, result_field):
""" Perform Pagure query """
result = []
url = "/".join((self.url, query))
while url:
log.debug("Pagure query: {0}".format(url))
try:
response = requests.get(url, headers=self.headers)
log.data("Response headers:\n{0}".format(response.headers))
except requests.RequestException as error:
log.error(error)
raise ReportError("Pagure search {0} failed.".format(self.url))
data = response.json()
objects = data[result_field]
log.debug("Result: {0} fetched".format(
listed(len(objects), "item")))
log.data(pretty(data))
# FIXME later: Work around https://pagure.io/pagure/issue/4057
if not objects:
break
result.extend(objects)
url = data[pagination]['next']
return result
def _fetch_activities(self):
""" Get organization activity, handle pagination """
activities = []
# Prepare url of the first page
url = '{0}/organizations/{1}/activity/'.format(
self.url, self.organization)
while url:
# Fetch one page of activities
try:
log.debug('Fetching activity data: {0}'.format(url))
response = requests.get(url, headers=self.headers)
if not response.ok:
log.error(response.text)
raise ReportError('Failed to fetch Sentry activities.')
data = response.json()
log.data("Response headers:\n{0}".format(
pretty(response.headers)))
log.debug("Fetched {0}.".format(listed(len(data), 'activity')))
log.data(pretty(data))
for activity in [Activity(item) for item in data]:
# We've reached the last page, older records not relevant
if activity.created < self.stats.options.since.date:
return activities
# Store only relevant activites (before until date)
if activity.created < self.stats.options.until.date:
log.details("Activity: {0}".format(activity))
activities.append(activity)
except requests.RequestException as error:
log.debug(error)
raise ReportError(
'Failed to fetch Sentry activities from {0}'.format(url))
# Check for possible next page
try:
url = NEXT_PAGE.search(response.headers['Link']).groups()[0]
except AttributeError:
url = None
return activities
