def _gather_logs(self):
"""Gather log events.
Returns:
list: A list of dictionaries containing log events.
"""
url = '{}{}'.format(self._SLACK_API_BASE_URL, self._endpoint())
headers = {
'Content-Type': 'application/x-www-form-urlencoded',
'Authorization':
'Bearer {}'.format(self._config.auth['auth_token']),
}
data = self._get_request_data()
success, response = self._make_post_request(url, headers, data, False)
if not success:
LOGGER.exception('Received bad response from slack')
return False
if not response.get('ok'):
LOGGER.exception('Received error or warning from slack')
return False
self._more_to_poll = self._check_for_more_to_poll(response)
results = self._filter_response_entries(response)
self._last_timestamp = int(time.time())
return results
def _perform_request(allow_retry=True):
try:
# Get the events using a make_request call with the box api. This is to
# support custom parameters such as 'created_after' and 'created_before'
box_response = self._client.make_request(
'GET',
self._client.get_url('events'),
params=params,
timeout=self._DEFAULT_REQUEST_TIMEOUT)
except BoxException:
LOGGER.exception('Failed to get events for %s', self.type())
return False, None # Return a tuple to conform to return value of safe_timeout
except ConnectionError:
# In testing, the requests connection seemed to get reset for no
# obvious reason, and a simple retry once works fine so catch it
# and retry once, but after that return False
LOGGER.exception(
'Bad response received from host, will retry once')
if allow_retry:
return _perform_request(allow_retry=False)
return False, None # Return a tuple to conform to return value of safe_timeout
# Return a successful status and the JSON from the box response
# Return a tuple to conform to return value of safe_timeout
return True, box_response.json()
def _create_service(self):
"""GSuite requests must be signed with the keyfile
Returns:
bool: True if the Google API discovery service was successfully established or False
if any errors occurred during the creation of the Google discovery service,
"""
if self._activities_service:
LOGGER.debug('Service already instantiated for %s', self.type())
return True
creds = self._load_credentials(self._config.auth['keyfile'])
if not creds:
return False
try:
resource = discovery.build('admin', 'reports_v1', credentials=creds)
except errors.Error:
LOGGER.exception('Failed to build discovery service for %s', self.type())
return False
# The google discovery service 'Resource' class that is returned by
# 'discovery.build' dynamically loads methods/attributes, so pylint will complain
# about no 'activities' member existing without the below pylint comment
self._activities_service = resource.activities() # pylint: disable=no-member
return True
def _generate_auth(self, hostname, params):
"""Duo requests must be signed each time.
This has been largely borrowed/updated from here:
https://github.com/duosecurity/duo_client_python/blob/master/duo_client/admin.py
"""
formatted_date = datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S -0000')
auth_string = '\n'.join([formatted_date, 'GET', hostname,
self._endpoint(), urllib.urlencode(params)])
try:
signature = hmac.new(self._config.auth['secret_key'],
auth_string, hashlib.sha1)
except TypeError:
LOGGER.exception('Could not generate hmac signature')
return False
# Format the basic auth with integration key and the hmac hex digest
basic_auth = ':'.join([self._config.auth['integration_key'],
signature.hexdigest()])
return {
'Date': formatted_date,
'Authorization': 'Basic {}'.format(b64encode(basic_auth)),
'Host': hostname
}
def _get_duo_logs(self, hostname, full_url):
"""Get all logs from the endpoint for this timeframe
Returns:
[
{
'timestamp': <int:unix timestamp>,
'device': <str:device>,
'username': <str:username>,
'factor': <str:factor>,
'result': <str:result>,
'ip': <str:ip address>,
'new_enrollment': <bool:if event corresponds to enrollment>,
'integration': <str:integration>,
'location': {
'state': '<str:state>',
'city': '<str:city>',
'country': '<str:country>'
}
}
]
"""
# Get the last timestamp and add one to it to avoid duplicates
# Sanity check mintime as unix timestamp, then transform to string
params = {'mintime': str(int(self._last_timestamp + 1))}
# Contstruct the headers for this request. Every request must be signed
headers = self._generate_auth(hostname, params)
if not headers:
return False
try:
# Make the request to the api, resulting in a bool or dict
result, response = self._make_get_request(full_url,
headers=headers,
params=params)
except requests.exceptions.ConnectionError:
LOGGER.exception('Received bad response from duo')
return False
if not result:
return False
# Duo stores the list of logs in the 'response' key of the response
logs = response['response']
if not logs:
LOGGER.info('No logs in response from duo')
return False
# Get the timestamp from the latest event. Duo produces these sequentially
# so we can just extract the timestamp from the last item in the list
self._last_timestamp = logs[-1]['timestamp']
# Check if the max amount of logs was returned with this request. If the value
# is not the max, then we are done polling logs for this timeframe
# Setting _more_to_poll to true here will allow the caller to try to poll again
self._more_to_poll = len(logs) >= self._MAX_RESPONSE_LOGS
# Return the list of logs to the caller so they can be send to the batcher
return logs
def _make_get_request():
# To use closure here is to make backoff logic patchable and testable.
try:
# Log GET request URL for debugging purpose, especially useful when
# debugging query syntax
LOGGER.debug('URL of GET request is %s', full_url)
resp = requests.get(full_url,
headers=headers,
params=params,
timeout=self._TIMEOUT)
# Return false if resp contains non-200 status code.
if not self._validate_status_code(resp):
return False, None
# When querying list of api versions and log files, Salesforce responses
# json content.
return True, resp.json()
except requests.exceptions.Timeout:
LOGGER.exception(
'Request timed out for when sending get request to %s',
full_url)
return False, None
except ValueError:
# When fetch log events, Salesforce returns raw data in csv format, not json
return True, resp.text.encode('utf-8')
def _gather_logs(self):
"""Gather the G Suite Admin Report logs for this application type
Returns:
bool or list: If the execution fails for some reason, return False.
Otherwise, return a list of activies for this application type.
"""
if not self._create_service():
return False
# Cache the last event timestamp so it can be used for future requests
if not self._next_page_token:
self._last_event_timestamp = self._last_timestamp
LOGGER.debug('Querying activities since %s for %s',
self._last_event_timestamp, self.type())
LOGGER.debug('Using next page token: %s', self._next_page_token)
activities_list = self._activities_service.list(
userKey='all',
applicationName=self._type(),
startTime=self._last_event_timestamp,
pageToken=self._next_page_token)
try:
results = activities_list.execute()
except self._GOOGLE_API_EXCEPTIONS:
LOGGER.exception('Failed to execute activities listing for %s',
self.type())
return False
if not results:
LOGGER.error(
'No results received from the G Suite API request for %s',
self.type())
return False
activities = results.get('items', [])
if not activities:
LOGGER.info('No logs in response from G Suite API request for %s',
self.type())
return False
# The activity api returns logs in reverse chronological order, for some reason, and
# therefore the newest log will be first in the list. This should only be updated
# once during the first poll
if not self._next_page_token:
self._last_timestamp = activities[0]['id']['time']
LOGGER.debug('Caching last timestamp: %s', self._last_timestamp)
self._next_page_token = results.get('nextPageToken')
self._more_to_poll = bool(self._next_page_token)
return activities
def _load_auth(cls, auth_data):
"""Load JWTAuth from Box service account JSON keyfile
Args:
auth_data (dict): The loaded keyfile data from a Box service account
JSON file
Returns:
boxsdk.JWTAuth Instance of JWTAuth that allows the client to authenticate
or False if there was an issue loading the auth
"""
try:
auth = JWTAuth.from_settings_dictionary(auth_data)
except (TypeError, ValueError, KeyError):
LOGGER.exception('Could not load JWT from settings dictionary')
return False
return auth
def _load_credentials(cls, keydata):
"""Load ServiceAccountCredentials from Google service account JSON keyfile
Args:
keydata (dict): The loaded keyfile data from a Google service account
JSON file
Returns:
oauth2client.service_account.ServiceAccountCredentials: Instance of
service account credentials for this discovery service
"""
try:
creds = ServiceAccountCredentials.from_json_keyfile_dict(
keydata, scopes=cls._SCOPES)
except (ValueError, KeyError):
# This has the potential to raise errors. See: https://tinyurl.com/y8q5e9rm
LOGGER.exception('Could not generate credentials from keyfile for %s',
cls.type())
return False
return creds
def _gather_logs(self):
"""Gather the G Suite Admin Report logs for this application type
Returns:
bool or list: If the execution fails for some reason, return False.
Otherwise, return a list of activies for this application type.
"""
if not self._create_service():
return False
activities_list = self._activities_service.list(
userKey='all',
applicationName=self._type(),
startTime=self._last_timestamp,
pageToken=self._next_page_token if self._next_page_token else None)
try:
results = activities_list.execute()
except errors.HttpError:
LOGGER.exception('Failed to execute activities listing')
return False
if not results:
LOGGER.error(
'No results received from the G Suite API request for %s',
self.type())
return False
self._next_page_token = results.get('nextPageToken')
self._more_to_poll = bool(self._next_page_token)
activities = results.get('items', [])
if not activities:
LOGGER.info('No logs in response from G Suite API request for %s',
self.type())
return False
self._last_timestamp = activities[-1]['id']['time']
return activities
def _fetch_event_logs(self, log_file_path):
"""Get event logs by sending GET request to each log file location.
Args:
log_file_path (str): log file location.
Returns:
list: a list of event logs or None.
"""
url = '{}/{}'.format(self._instance_url, log_file_path)
try:
success, resp = self._make_get_request(url, self._auth_headers)
except SalesforceAppError:
LOGGER.exception('Failed to get event logs')
return
if not (success and resp):
LOGGER.error('Failed to get event logs')
return
# skip header line before passing to rule processor
return resp.splitlines()[1:]
def _wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except requests.exceptions.Timeout:
LOGGER.exception('Request timed out for %s', args[0].type())
return False, None