def pulls(self, from_date=None, to_date=None):
"""Fetch the pull requests from the repository.
The method retrieves, from a BitBucket repository, the pull requests
from a v1 bitbucket server.
:returns: a generator of pull requests
"""
# ?sort=-updated_on&state=ALL&q=updated_on>=2014-03-26T13:15:00.000000+AND+updated_on<=2017-08-26T13:15:00.000000
payload = {
'sort': '-updated_on',
'state': 'ALL',
'pagelen': self.max_items
}
tmp_from_date = re.sub('(\+.*)?', '', str(from_date)).replace(" ", "T")
tmp_to_date = re.sub('(\+.*)?', '', str(to_date)).replace(" ", "T")
if from_date and to_date:
if from_date < to_date:
payload[
'q'] = 'updated_on>=' + tmp_from_date + '+AND+updated_on<=' + tmp_to_date
else:
logger.warning(
"from_date is set to a later datetime than to_date: %s > %s",
(tmp_from_date, tmp_to_date))
return
elif from_date:
payload['q'] = 'updated_on>=' + tmp_from_date
elif to_date:
payload['q'] = 'updated_on<=' + tmp_to_date
payload_str = "&".join("%s=%s" % (k, v) for k, v in payload.items())
path = urijoin("pullrequests")
return self.fetch_items(path, payload_str)
def _fetch(self, resource, params):
"""Fetch a resource.
Method to fetch and to iterate over the contents of a
type of resource. The method returns a generator of
pages for that resource and parameters.
:param resource: type of the resource
:param params: parameters to filter
:returns: a generator of pages for the requeste resource
"""
url = urijoin(self.base_url, resource)
params[self.PKEY] = self.api_key
params[self.PSIGN] = 'true',
do_fetch = True
while do_fetch:
logger.debug("Meetup client calls resource: %s params: %s",
resource, str(params))
if not self.from_archive:
self.sleep_for_rate_limit()
r = self.fetch(url, payload=params)
if not self.from_archive:
self.update_rate_limit(r)
yield r.text
if r.links and 'next' in r.links:
url = r.links['next']['url']
params = {self.PKEY: self.api_key, self.PSIGN: 'true'}
else:
do_fetch = False
def fetch_items(self, path, payload):
"""Return the items from GitLab API using links pagination"""
page = 0 # current page
last_page = None # last page
url_next = urijoin(self.base_url, self.RPROJECTS,
self.owner + '%2F' + self.repository, path)
logger.debug("Get GitLab paginated items from " + url_next)
response = self.fetch(url_next, payload=payload)
response.encoding = 'utf-8'
items = response.text
page += 1
if 'last' in response.links:
last_url = response.links['last']['url']
last_page = last_url.split('&page=')[1].split('&')[0]
last_page = int(last_page)
logger.debug("Page: %i/%i" % (page, last_page))
while items:
yield items
items = None
if 'next' in response.links:
url_next = response.links['next']['url'] # Loving requests :)
response = self.fetch(url_next, payload=payload)
page += 1
items = response.text
if not last_page:
logger.debug("Page: %i" % page)
else:
logger.debug("Page: %i/%i" % (page, last_page))
def user_orgs(self, login):
"""Get the user public organizations"""
if login in self._users_orgs:
return self._users_orgs[login]
url = urijoin(self.base_url, self.RUSERS, login, self.RORGS)
try:
r = self.fetch(url)
orgs = r.text
except requests.exceptions.HTTPError as error:
# 404 not found is wrongly received sometimes
if error.response.status_code == 404:
logger.error("Can't get github login orgs: %s", error)
orgs = '[]'
else:
raise error
except requests.exceptions.RetryError as error:
logger.error("Can't get github login orgs: %s", error)
orgs = '[]'
self._users_orgs[login] = orgs
return orgs
def _update_current_rate_limit(self):
"""Update rate limits data for the current token"""
url = urijoin(self.base_url, self.RRATE_LIMIT)
try:
# Turn off archiving when checking rates, because that would cause
# archive key conflict (the same URLs giving different responses)
arch = self.archive
self.archive = None
response = super().fetch(url)
self.archive = arch
self.update_rate_limit(response)
self.last_rate_limit_checked = self.rate_limit
except requests.exceptions.HTTPError as error:
if error.response.status_code == 404:
logger.warning("Rate limit not initialized: %s", error)
elif error.response.status_code == 401:
logger.debug(
"GitHub APP with {} ID: access token expired, creating new one"
.format(self.github_app_id))
self._update_access_token()
else:
raise error
def __init__(self,
url,
channel,
api_token,
max_items=MAX_ITEMS,
tag=None,
archive=None,
sleep_for_rate=False,
min_rate_to_sleep=MIN_RATE_LIMIT,
sleep_time=DEFAULT_SLEEP_TIME):
origin = urijoin(url, channel)
super().__init__(origin, tag=tag, archive=archive)
self.url = url
self.channel = channel
self.api_token = api_token
self.max_items = max_items
self.sleep_for_rate = sleep_for_rate
self.min_rate_to_sleep = min_rate_to_sleep
self.sleep_time = sleep_time
self.client = None
self._users = {}
def issues(self, from_date=None):
"""Fetch the issues from the repository.
The method retrieves, from a GitHub repository, the issues
updated since the given date.
:param from_date: obtain issues updated since this date
:returns: a generator of issues
"""
payload = {
'state': 'all',
'per_page': self.max_items,
'direction': 'asc',
'sort': 'updated'
}
if from_date:
payload['since'] = from_date.isoformat()
path = urijoin("issues")
# 调用
return self.fetch_items(path, payload)
def __init__(self,
owner=None,
repository=None,
api_token=None,
base_url=None,
tag=None,
archive=None,
sleep_for_rate=False,
min_rate_to_sleep=MIN_RATE_LIMIT,
max_retries=MAX_RETRIES,
sleep_time=DEFAULT_SLEEP_TIME,
max_items=MAX_CATEGORY_ITEMS_PER_PAGE,
ssl_verify=True):
if api_token is None:
api_token = []
origin = base_url if base_url else GITHUB_URL
origin = urijoin(origin, owner, repository)
super().__init__(origin,
tag=tag,
archive=archive,
ssl_verify=ssl_verify)
self.owner = owner
self.repository = repository
self.api_token = api_token
self.base_url = base_url
self.sleep_for_rate = sleep_for_rate
self.min_rate_to_sleep = min_rate_to_sleep
self.max_retries = max_retries
self.sleep_time = sleep_time
self.max_items = max_items
self.client = None
self.exclude_user_data = False
self._users = {} # internal users cache
def user(self, login):
"""Get the user information and update the user cache"""
user = None
if login in self._users:
return self._users[login]
url_user = urijoin(self.base_url, self.RUSERS, login)
logger.debug("Getting info for %s" % url_user)
try:
r = self.fetch(url_user)
user = r.text
self._users[login] = user
except requests.exceptions.HTTPError as error:
# When the login is no longer exist or the token has no permission
if error.response.status_code == 404:
logger.error("Can't get github login: %s", error)
user = '******'
else:
raise error
return user
def __init__(self, owner=None, repository=None, api_token=None,
is_oauth_token=False, base_url=None, tag=None, archive=None,
sleep_for_rate=False, min_rate_to_sleep=MIN_RATE_LIMIT,
max_retries=MAX_RETRIES, sleep_time=DEFAULT_SLEEP_TIME,
blacklist_ids=None):
origin = base_url if base_url else GITLAB_URL
origin = urijoin(origin, owner, repository)
if not api_token and is_oauth_token:
raise BackendError(cause="is_oauth_token is True but api_token is None")
super().__init__(origin, tag=tag, archive=archive)
self.base_url = base_url
self.owner = owner
self.repository = repository
self.api_token = api_token
self.is_oauth_token = is_oauth_token
self.sleep_for_rate = sleep_for_rate
self.min_rate_to_sleep = min_rate_to_sleep
self.max_retries = max_retries
self.sleep_time = sleep_time
self.blacklist_ids = blacklist_ids
self.client = None
self._users = {} # internal users cache
def pulls(self, from_date=None):
"""Fetch the pull requests from the repository.
The method retrieves, from a GitHub repository, the pull requests
updated since the given date.
:param from_date: obtain pull requests updated since this date
:returns: a generator of pull requests
"""
issues_groups = self.issues(from_date=from_date)
for raw_issues in issues_groups:
issues = json.loads(raw_issues)
for issue in issues:
if "pull_request" not in issue:
continue
pull_number = issue["number"]
path = urijoin(self.base_url, 'repos', self.owner, self.repository, "pulls", pull_number)
r = self.fetch(path)
pull = r.text
yield pull
def get_questions(self, offset=None):
"""Retrieve questions from older to newer updated starting offset"""
page = KitsuneClient.FIRST_PAGE
if offset:
page += int(offset / KitsuneClient.ITEMS_PER_PAGE)
while True:
api_questions_url = urijoin(self.base_url, '/question') + '/'
params = {
"page": page,
"ordering": "updated"
}
questions = self.fetch(api_questions_url, params)
yield questions
questions_json = json.loads(questions)
next_uri = questions_json['next']
if not next_uri:
break
page += 1
def get_api_questions(self, path):
"""Retrieve a question page using the API.
:param page: page to retrieve
"""
npages = 1
next_request = True
path = urijoin(self.base_url, path)
while next_request:
try:
params = {
'page': npages,
'sort': self.ORDER_API
}
response = self.fetch(path, payload=params)
whole_page = response.text
raw_questions = json.loads(whole_page)
tpages = raw_questions['pages']
logger.debug("Fetching questions from '%s': page %s/%s",
self.base_url, npages, tpages)
if npages == tpages:
next_request = False
npages = npages + 1
yield raw_questions
except requests.exceptions.TooManyRedirects as e:
logger.warning("%s, data not retrieved for resource %s", e, path)
next_request = False
def __configure_kibiter_old(self, kibiter_major):
if 'panels' not in self.conf:
logger.warning(
"Panels config not availble. Not configuring Kibiter.")
return False
kibiter_time_from = self.conf['panels']['kibiter_time_from']
kibiter_default_index = self.conf['panels']['kibiter_default_index']
logger.info(
"Configuring Kibiter %s for default index %s and time frame %s",
kibiter_major, kibiter_default_index, kibiter_time_from)
kibiter_version = self.__kibiter_version()
if not kibiter_version:
return False
logger.info("Kibiter/Kibana: version found is %s" % kibiter_version)
time_picker = "{\n \"from\": \"" + kibiter_time_from \
+ "\",\n \"to\": \"now\",\n \"mode\": \"quick\"\n}"
config_resource = '.kibana/config/' + kibiter_version
kibiter_config = {
"defaultIndex": kibiter_default_index,
"timepicker:timeDefaults": time_picker
}
es_url = self.conf['es_enrichment']['url']
url = urijoin(es_url, config_resource)
res = self.grimoire_con.post(url,
data=json.dumps(kibiter_config),
headers=ES6_HEADER)
res.raise_for_status()
logger.info("Kibiter settings configured!")
return True
def messages(self, channel, from_date, offset):
"""Fetch messages from a channel.
The messages are fetch in ascending order i.e. from the oldest
to the latest based on the time they were last updated. A query is
also passed as a param to fetch the messages from a given date.
"""
query = '{"_updatedAt": {"$gte": {"$date": "%s"}}}' % from_date.isoformat(
)
# The 'sort' param accepts a field based on which the messages are sorted.
# The value of the field can be 1 for ascending order or -1 for descending order.
params = {
"roomName": channel,
"sort": '{"_updatedAt": 1}',
"count": self.max_items,
"offset": offset,
"query": query
}
path = urijoin(self.base_url, self.RCHANNEL_MESSAGES)
response = self.fetch(path, params)
return response
def results(self, from_date, to_date=None):
"""Get test cases results."""
fdt = from_date.strftime("%Y-%m-%d %H:%M:%S")
params = {self.PFROM_DATE: fdt, self.PPAGE: 1}
if to_date:
tdt = to_date.strftime("%Y-%m-%d %H:%M:%S")
params[self.PTO_DATE] = tdt
while True:
url = urijoin(self.base_url, self.FUNCTEST_API_PATH, self.RRESULTS)
response = self.fetch(url, payload=params)
content = response.text
yield content
j = json.loads(content)
page = j['pagination']['current_page']
total_pages = j['pagination']['total_pages']
if page >= total_pages:
break
params[self.PPAGE] = page + 1
def __init__(self,
owner,
repository,
tokens,
base_url=None,
sleep_for_rate=False,
min_rate_to_sleep=MIN_RATE_LIMIT,
sleep_time=DEFAULT_SLEEP_TIME,
max_retries=MAX_RETRIES,
archive=None,
from_archive=False):
self.owner = owner
self.repository = repository
self.tokens = tokens
self.n_tokens = len(self.tokens)
self.current_token = None
self.last_rate_limit_checked = None
if base_url:
base_url = urijoin(base_url, 'api', 'v3')
else:
base_url = GITHUB_API_URL
super().__init__(base_url,
sleep_time=sleep_time,
max_retries=max_retries,
extra_headers=self._set_extra_headers(),
extra_status_forcelist=self.EXTRA_STATUS_FORCELIST,
archive=archive,
from_archive=from_archive)
super().setup_rate_limit_handler(sleep_for_rate=sleep_for_rate,
min_rate_to_sleep=min_rate_to_sleep)
# Choose best API token (with maximum API points remaining)
if not self.from_archive:
self._choose_best_api_token()
def __init__(self,
group=None,
room=None,
api_token=None,
max_items=MAX_ITEMS,
sleep_for_rate=False,
min_rate_to_sleep=MIN_RATE_LIMIT,
tag=None,
archive=None,
ssl_verify=True):
origin = urijoin(GITTER_URL, group, room)
super().__init__(origin,
tag=tag,
archive=archive,
ssl_verify=ssl_verify)
self.group = group
self.room = room
self.api_token = api_token
self.max_items = max_items
self.sleep_for_rate = sleep_for_rate
self.min_rate_to_sleep = min_rate_to_sleep
self.client = None
self.room_id = None
def test_initialization(self):
"""Test whether attributes are initializated"""
dockerhub = DockerHub('grimoirelab', 'perceval', tag='test')
expected_origin = urijoin(DOCKERHUB_URL, 'grimoirelab', 'perceval')
self.assertEqual(dockerhub.owner, 'grimoirelab')
self.assertEqual(dockerhub.repository, 'perceval')
self.assertEqual(dockerhub.origin, expected_origin)
self.assertEqual(dockerhub.tag, 'test')
self.assertIsNone(dockerhub.client)
self.assertTrue(dockerhub.ssl_verify)
# When tag is empty or None it will be set to
# the value in
dockerhub = DockerHub('grimoirelab', 'perceval', ssl_verify=False)
self.assertEqual(dockerhub.origin, expected_origin)
self.assertEqual(dockerhub.tag, expected_origin)
self.assertFalse(dockerhub.ssl_verify)
dockerhub = DockerHub('grimoirelab', 'perceval', tag='')
self.assertEqual(dockerhub.origin, expected_origin)
self.assertEqual(dockerhub.tag, expected_origin)
def get_messages(self, anchor):
"""Fetch the messages."""
params = {
'anchor':
'{}'.format(anchor),
'num_before':
'0',
'num_after':
'2',
'apply_markdown':
'false',
'narrow':
json.dumps([{
'operator': 'stream',
'operand': '{}'.format(self.stream)
}])
}
path = urijoin(self.url, "/api/v1/messages")
r = self.fetch(path, payload=params, auth=(self.email, self.api_token))
return r.text
def export_dashboard(self, dashboard_id):
"""Export a dashboard identified by its ID.
This method returns a dashboard based on `dashboard_id` using the endpoint
`export` of the Dashboard API.
A `DataExportError` is thrown if the API returned an error message
or if a 400 HTTP error occurred. Other HTTP errors are not incapsulated and returned
as they are.
:param dashboard_id: ID of the dashboard
:returns the dashboard exported
"""
url = urijoin(self.base_url, self.API_DASHBOARDS_URL,
self.API_EXPORT_COMMAND + '?dashboard=' + dashboard_id)
try:
dashboard = self.fetch(url)
errors = [obj for obj in dashboard['objects'] if 'error' in obj]
if errors:
msg = errors[0]
cause = "Impossible to export dashboard with id %s, %s" % (
dashboard_id, msg)
logger.error(cause)
raise DataExportError(cause=cause)
except requests.exceptions.HTTPError as error:
if error.response.status_code == 400:
cause = "Impossible to export dashboard with id %s" % dashboard_id
logger.error(cause)
raise DataExportError(cause=cause)
else:
raise error
return dashboard
import json
import logging
from grimoirelab_toolkit.datetime import datetime_utcnow
from grimoirelab_toolkit.uris import urijoin
from ...backend import (Backend,
BackendCommand,
BackendCommandArgumentParser)
from ...client import HttpClient
CATEGORY_DOCKERHUB_DATA = "dockerhub-data"
DOCKERHUB_URL = "https://hub.docker.com/"
DOCKERHUB_API_URL = urijoin(DOCKERHUB_URL, 'v2')
DOCKER_OWNER = 'library'
DOCKER_SHORTCUT_OWNER = '_'
logger = logging.getLogger(__name__)
class DockerHub(Backend):
"""DockerHub backend for Perceval.
This class retrieves data from a repository stored
in the Docker Hub site. To initialize this class owner
and repositories where data will be fetched must be provided.
The origin of the data will be built with both parameters.
def __init__(self, url, archive=None, from_archive=False):
super().__init__(urijoin(url, "api.php"), archive=archive, from_archive=from_archive)
self.limit = "max" # Always get the max number of items
def pull_requested_reviewers(self, pr_number):
"""Get pull requested reviewers"""
requested_reviewers_url = urijoin("pulls", str(pr_number),
"requested_reviewers")
return self.fetch_items(requested_reviewers_url, {})
class MattermostClient(HttpClient, RateLimitHandler):
"""Mattermost API client.
Client for fetching information from a Mattermost server
using its REST API.
:param base_url: URL of the Mattermost server
:param api_key: key needed to use the API
:param max_items: maximum number of items fetched per request
:param sleep_for_rate: sleep until rate limit is reset
:param min_rate_to_sleep: minimun rate needed to sleep until
it will be reset
:param sleep_time: time (in seconds) to sleep in case
of connection problems
:param archive: an archive to store/read fetched data
:param from_archive: it tells whether to write/read the archive
"""
API_URL = urijoin('%(base_url)s', 'api', 'v4', '%(entrypoint)s')
RCHANNELS = 'channels'
RPOSTS = 'posts'
RUSERS = 'users'
PCHANNEL_ID = 'channel_id'
PPAGE = 'page'
PPER_PAGE = 'per_page'
def __init__(self, base_url, api_token, max_items=MAX_ITEMS,
sleep_for_rate=False, min_rate_to_sleep=MIN_RATE_LIMIT,
sleep_time=DEFAULT_SLEEP_TIME,
archive=None, from_archive=False):
self.api_token = api_token
self.max_items = max_items
super().__init__(base_url.rstrip('/'),
sleep_time=sleep_time,
extra_headers=self._set_extra_headers(),
archive=archive, from_archive=from_archive)
super().setup_rate_limit_handler(sleep_for_rate=sleep_for_rate,
min_rate_to_sleep=min_rate_to_sleep)
def channel(self, channel):
"""Fetch the channel information"""
entrypoint = self.RCHANNELS + '/' + channel
params = {
self.PCHANNEL_ID: channel
}
response = self._fetch(entrypoint, params)
return response
def posts(self, channel, page=None):
"""Fetch the history of a channel."""
entrypoint = self.RCHANNELS + '/' + channel + '/' + self.RPOSTS
params = {
self.PPER_PAGE: self.max_items
}
if page is not None:
params[self.PPAGE] = page
response = self._fetch(entrypoint, params)
return response
def user(self, user):
"""Fetch user data."""
entrypoint = self.RUSERS + '/' + user
response = self._fetch(entrypoint, None)
return response
def fetch(self, url, payload=None, headers=None,
method=HttpClient.GET, stream=False, verify=True):
"""Override fetch method to handle API rate limit.
:param url: link to the resource
:param payload: payload of the request
:param headers: headers of the request
:param method: type of request call (GET or POST)
:param stream: defer downloading the response body until the response
content is available
:returns a response object
"""
if not self.from_archive:
self.sleep_for_rate_limit()
response = super().fetch(url, payload, headers, method, stream, verify)
if not self.from_archive:
self.update_rate_limit(response)
return response
def calculate_time_to_reset(self):
"""Number of seconds to wait.
The time is obtained by the different between the current date
and the next date when the token is fully regenerated.
"""
current_epoch = datetime_utcnow().replace(microsecond=0).timestamp() + 1
time_to_reset = self.rate_limit_reset_ts - current_epoch
if time_to_reset < 0:
time_to_reset = 0
return time_to_reset
def _fetch(self, entry_point, params):
"""Fetch a resource.
:param entrypoint: entrypoint to access
:param params: dict with the HTTP parameters needed to access the
given entry point
"""
url = self.API_URL % {'base_url': self.base_url, 'entrypoint': entry_point}
logger.debug("Mattermost client requests: %s params: %s",
entry_point, str(params))
r = self.fetch(url, payload=params)
return r.text
def _set_extra_headers(self):
"""Set authentication tokens."""
headers = {
'Authorization': 'Bearer ' + self.api_token
}
return headers
class SlackClient(HttpClient):
"""Slack API client.
Client for fetching information from the Slack server
using its REST API.
:param api_token: key needed to use the API
:param max_items: maximum number of items per request
:param archive: an archive to store/read fetched data
:param from_archive: it tells whether to write/read the archive
:param ssl_verify: enable/disable SSL verification
"""
URL = urijoin(SLACK_URL, 'api', '%(resource)s')
AUTHORIZATION_HEADER = 'Authorization'
RCONVERSATION_INFO = 'conversations.members'
RCHANNEL_INFO = 'channels.info'
RCHANNEL_HISTORY = 'channels.history'
RUSER_INFO = 'users.info'
PCHANNEL = 'channel'
PCOUNT = 'count'
POLDEST = 'oldest'
PLATEST = 'latest'
PTOKEN = 'token'
PUSER = '******'
def __init__(self, api_token, max_items=MAX_ITEMS, archive=None, from_archive=False, ssl_verify=True):
super().__init__(SLACK_URL, archive=archive, from_archive=from_archive, ssl_verify=ssl_verify)
self.api_token = api_token
self.max_items = max_items
def conversation_members(self, conversation):
"""Fetch the number of members in a conversation, which is a supertype for public and
private ones, DM and group DM.
:param conversation: the ID of the conversation
"""
members = 0
resource = self.RCONVERSATION_INFO
params = {
self.PCHANNEL: conversation,
}
raw_response = self._fetch(resource, params)
response = json.loads(raw_response)
members += len(response["members"])
while 'next_cursor' in response['response_metadata'] and response['response_metadata']['next_cursor']:
params['cursor'] = response['response_metadata']['next_cursor']
raw_response = self._fetch(resource, params)
response = json.loads(raw_response)
members += len(response["members"])
return members
def channel_info(self, channel):
"""Fetch information about a channel."""
resource = self.RCHANNEL_INFO
params = {
self.PCHANNEL: channel,
}
response = self._fetch(resource, params)
return response
def history(self, channel, oldest=None, latest=None):
"""Fetch the history of a channel."""
resource = self.RCHANNEL_HISTORY
params = {
self.PCHANNEL: channel,
self.PCOUNT: self.max_items
}
if oldest is not None:
formatted_oldest = self.__format_timestamp(oldest, subtract=True)
params[self.POLDEST] = formatted_oldest
if latest is not None:
formatted_latest = self.__format_timestamp(latest)
params[self.PLATEST] = formatted_latest
response = self._fetch(resource, params)
return response
def user(self, user_id):
"""Fetch user info."""
resource = self.RUSER_INFO
params = {
self.PUSER: user_id
}
response = self._fetch(resource, params)
return response
@staticmethod
def sanitize_for_archive(url, headers, payload):
"""Sanitize payload of a HTTP request by removing the token information
before storing/retrieving archived items
:param: url: HTTP url request
:param: headers: HTTP headers request
:param: payload: HTTP payload request
:returns url, headers and the sanitized payload
"""
if SlackClient.AUTHORIZATION_HEADER in headers:
headers.pop(SlackClient.AUTHORIZATION_HEADER)
return url, headers, payload
def _fetch(self, resource, params):
"""Fetch a resource.
:param resource: resource to get
:param params: dict with the HTTP parameters needed to get
the given resource
"""
url = self.URL % {'resource': resource}
headers = {
self.AUTHORIZATION_HEADER: 'Bearer {}'.format(self.api_token)
}
logger.debug("Slack client requests: %s params: %s",
resource, str(params))
r = self.fetch(url, payload=params, headers=headers)
# Check for possible API errors
result = r.json()
if not result['ok']:
raise SlackClientError(error=result['error'])
return r.text
def __format_timestamp(self, ts, subtract=False):
"""Handle the timestamp value to be passed to the channels.history API endpoint. In
particular, two cases are covered:
- Since the minimum value supported by Slack is 0, the value 0.0 must be converted.
- Slack does not include in its result the lower limit of the search if it has
the same date of 'oldest'. To get this messages too, we subtract a low value to
be sure the dates are not the same. To avoid precision problems it is subtracted
by five decimals and not by six.
:param ts: timestamp float value
:param subtract: if True, `ts` is decreased by 0.00001
"""
if ts == 0.0:
return "0"
processed = ts
if processed > 0.0 and subtract:
processed -= .00001
processed = FLOAT_FORMAT.format(processed)
return processed
def __get_url(self, path):
"""Build genereic URL"""
return urijoin(self.base_url, path)
def __get_url_distribution_package(self):
"""Build URL distribution package"""
return urijoin(self.__get_url_distribution(), self.RSOURCE,
self.package)
def __get_url_distribution(self):
"""Build URL distribution"""
return urijoin(self.base_url, self.distribution)
class SlackClient(HttpClient):
"""Slack API client.
Client for fetching information from the Slack server
using its REST API.
:param api_key: key needed to use the API
:param max_items: maximum number of items per request
:param archive: an archive to store/read fetched data
:param from_archive: it tells whether to write/read the archive
"""
URL = urijoin(SLACK_URL, 'api', '%(resource)s')
RCHANNEL_INFO = 'channels.info'
RCHANNEL_HISTORY = 'channels.history'
RUSER_INFO = 'users.info'
PCHANNEL = 'channel'
PCOUNT = 'count'
POLDEST = 'oldest'
PLATEST = 'latest'
PTOKEN = 'token'
PUSER = '******'
def __init__(self, api_token, max_items=MAX_ITEMS, archive=None, from_archive=False):
super().__init__(SLACK_URL, archive=archive, from_archive=from_archive)
self.api_token = api_token
self.max_items = max_items
def channel_info(self, channel):
"""Fetch information about a channel."""
resource = self.RCHANNEL_INFO
params = {
self.PCHANNEL: channel,
}
response = self._fetch(resource, params)
return response
def history(self, channel, oldest=None, latest=None):
"""Fetch the history of a channel."""
resource = self.RCHANNEL_HISTORY
params = {
self.PCHANNEL: channel,
self.PCOUNT: self.max_items
}
if oldest is not None:
params[self.POLDEST] = oldest
if latest is not None:
params[self.PLATEST] = latest
response = self._fetch(resource, params)
return response
def user(self, user_id):
"""Fetch user info."""
resource = self.RUSER_INFO
params = {
self.PUSER: user_id
}
response = self._fetch(resource, params)
return response
@staticmethod
def sanitize_for_archive(url, headers, payload):
"""Sanitize payload of a HTTP request by removing the token information
before storing/retrieving archived items
:param: url: HTTP url request
:param: headers: HTTP headers request
:param: payload: HTTP payload request
:returns url, headers and the sanitized payload
"""
if SlackClient.PTOKEN in payload:
payload.pop(SlackClient.PTOKEN)
return url, headers, payload
def _fetch(self, resource, params):
"""Fetch a resource.
:param resource: resource to get
:param params: dict with the HTTP parameters needed to get
the given resource
"""
url = self.URL % {'resource': resource}
params[self.PTOKEN] = self.api_token
logger.debug("Slack client requests: %s params: %s",
resource, str(params))
r = self.fetch(url, payload=params)
# Check for possible API errors
result = r.json()
if not result['ok']:
raise SlackClientError(error=result['error'])
return r.text
