def __setitem__(self, key, value):
# Do some safety checking so we don't save a malformed state
if key == self._STATE_KEY and not getattr(self.States,
str(value).upper(), None):
LOGGER.error('Current state cannot be saved with value \'%s\'',
value)
return
dict.__setitem__(self, key, value)
# If this is a key related to the state config, save the state in parameter store
if key in self._state_keys():
self._save_state()
def _check_http_response(self, response):
"""Method for checking for a valid HTTP response code
Returns:
bool: Indicator of whether or not this request was successful
"""
success = response is not None and (200 <= response.status_code <= 299)
if response is not None and not success:
LOGGER.error('HTTP request failed for service \'%s\': [%d] %s',
self.type(), response.status_code,
response.json()['message'])
return success
def _request_token(self):
"""Request OAuth token from salesforce
Meanwhile, it will also get instance url which will be used in future
requests. The instance url identifies the Salesforce instance to which
API calls should be sent.
Returns:
bool: Returns True if update auth headers and instance url successfully.
"""
headers = {'Content-Type': 'application/x-www-form-urlencoded'}
# required credentials when request for a token.
data = {
'grant_type':
'password',
'client_id':
self._config.auth['client_id'],
'client_secret':
self._config.auth['client_secret'],
'username':
self._config.auth['username'],
'password':
'******'.format(self._config.auth['password'],
self._config.auth['security_token']),
'response_type':
'code',
'redirect_uri':
self._SALESFORCE_TOKEN_URL
}
success, response = self._make_post_request(self._SALESFORCE_TOKEN_URL,
headers, data, False)
if not (success and response):
return False
if not (response.get('access_token') and response.get('instance_url')):
LOGGER.error(
'Response invalid generating headers for service \'%s\'',
self._type())
return False
bearer = 'Bearer {}'.format(response.get('access_token'))
self._auth_headers = {
'Content-Type': 'application/json',
'Authorization': bearer
}
self._instance_url = response.get('instance_url')
LOGGER.debug('Successfully obtain OAuth token and instance URL')
return True
def _finalize(self):
"""Method for performing any final steps, like saving applicable state"""
self._config.mark_success()
if not self._last_timestamp:
LOGGER.error('Ending last timestamp is 0. This should not happen and is likely '
'due to the subclass not setting this value.')
if self._last_timestamp == self._config.start_last_timestamp:
LOGGER.error('Ending last timestamp is the same as the beginning last timestamp')
LOGGER.info('App complete for service \'%s\'. Gathered %d logs in %d polls.',
self.type(), self._gathered_log_count, self._poll_count)
self._config.last_timestamp = self._last_timestamp
def _send_logs_to_stream_alert(self, source_function, logs):
"""Protected method for sending logs to the rule processor lambda
function for processing. This performs some size checks before sending.
Args:
source_function (str): The app function name from which the logs came
logs (list): List of the logs that have been gathered
"""
# Create a payload to be sent to the rule processor that contains the
# service these logs were collected from and the list of logs
payload = {
'Records': [{
'stream_alert_app': source_function,
'logs': logs
}]
}
payload_json = json.dumps(payload, separators=(',', ':'))
if len(payload_json) > MAX_LAMBDA_PAYLOAD_SIZE:
LOGGER.debug(
'Log payload size for %d logs exceeds limit and will be '
'segmented (%d > %d max).', len(logs), len(payload_json),
MAX_LAMBDA_PAYLOAD_SIZE)
return False
LOGGER.debug('Sending %d logs to rule processor with payload size %d',
len(logs), len(payload_json))
try:
response = Batcher.LAMBDA_CLIENT.invoke(
FunctionName=self.rp_function,
InvocationType='Event',
Payload=payload_json,
Qualifier='production')
except ClientError as err:
LOGGER.error(
'An error occurred while sending logs to '
'\'%s:production\'. Error is: %s', self.rp_function,
err.response)
raise
LOGGER.info('Sent %d logs to \'%s\' with Lambda request ID \'%s\'',
len(logs), self.rp_function,
response['ResponseMetadata']['RequestId'])
return True
def _initialize(self):
"""Method for performing any startup steps, like setting state to running"""
# Perform another safety check to make sure this is not being invoked already
if self._config.is_running:
LOGGER.error('App already running for service \'%s\'.', self.type())
return False
LOGGER.info('App starting for service \'%s\'.', self.type())
# Validate the auth in the config. This raises an exception upon failure
self._validate_auth()
self._last_timestamp = self._config.last_timestamp
# Mark this app as running, which updates the parameter store
self._config.mark_running()
return True
def _gather_logs(self):
"""Gather the Box Admin Events
The ideal way to do this would be to use the boxsdk.events.Events class and the
`get_events` method to retrieve these events. However, this method does allow you
to pass keyword arguments (such as params) which are needed for specifying the
'created_after' parameter.
Returns:
bool or list: If the execution fails for some reason, return False.
Otherwise, return a list of box admin event entries.
"""
if not self._create_client():
LOGGER.error('Could not create box client for %s', self.type())
return False
result, response = self._make_request()
# If the result is False, errors would be previously logged up
# the stack before this, so just return False
if not result:
return False
if not response:
LOGGER.error('No results received from the Box API request for %s',
self.type())
return False
self._more_to_poll = int(
response['chunk_size']) >= self._MAX_CHUNK_SIZE
events = response.get('entries', [])
if not events:
LOGGER.info(
'No events in response from the Box API request for %s',
self.type())
return False
self._next_stream_position = response['next_stream_position']
self._last_timestamp = events[-1]['created_at']
return events
def _get_latest_api_version(self):
"""GET request to fetch supported API versions and find the latest API version
The example of response json body:
[
{
"version": "20.0",
"label": "Winter '11",
"url": "/services/data/v20.0"
},
{
"version": "21.0",
"label": "Spring '11",
"url": "/services/data/v21.0"
},
{
"version": "26.0",
"label": "Winter '13",
"url": "/services/data/v26.0"
}
]
Returns:
bool: Return True if get latest api version successfully.
"""
url = '{}/services/data/'.format(self._instance_url)
success, response = self._make_get_request(url, self._auth_headers)
if not (success and response):
LOGGER.error('Failed to fetch lastest api version')
return False
versions = [float(version.get('version', 0)) for version in response]
if versions:
self._latest_api_version = str(sorted(versions)[-1])
if self._latest_api_version == '0.0':
LOGGER.error('Failed to obtain latest API version')
return False
LOGGER.debug('Successfully obtain latest API version %s',
self._latest_api_version)
return True
def __setitem__(self, key, value):
# Do some safety checking so we don't save a malformed state
if key == self._STATE_KEY and not getattr(self.States, str(value).upper(), None):
LOGGER.error('Current state cannot be saved with value \'%s\'', value)
return
# Cache the old value to see if the new value differs
current_value = self.get(key)
dict.__setitem__(self, key, value)
# If this is a key related to the state config, save the state in parameter store
if key in self._state_keys() and current_value != value:
state_name = '_'.join([self['function_name'], self.STATE_CONFIG_SUFFIX])
param_value = json.dumps({key: self[key] for key in self._state_keys()})
try:
self._save_state(state_name, param_value)
except ClientError as err:
raise AppIntegrationStateError('Could not save current state to parameter '
'store with name \'{}\'. Response: '
'{}'.format(state_name, err.response))
def _gather_logs(self):
"""Gather the G Suite Admin Report logs for this application type
Returns:
bool or list: If the execution fails for some reason, return False.
Otherwise, return a list of activies for this application type.
"""
if not self._create_service():
return False
activities_list = self._activities_service.list(
userKey='all',
applicationName=self._type(),
startTime=self._last_timestamp,
pageToken=self._next_page_token if self._next_page_token else None)
try:
results = activities_list.execute()
except errors.HttpError:
LOGGER.exception('Failed to execute activities listing')
return False
if not results:
LOGGER.error(
'No results received from the G Suite API request for %s',
self.type())
return False
self._next_page_token = results.get('nextPageToken')
self._more_to_poll = bool(self._next_page_token)
activities = results.get('items', [])
if not activities:
LOGGER.info('No logs in response from G Suite API request for %s',
self.type())
return False
self._last_timestamp = activities[-1]['id']['time']
return activities
def _fetch_event_logs(self, log_file_path):
"""Get event logs by sending GET request to each log file location.
Args:
log_file_path (str): log file location.
Returns:
list: a list of event logs or None.
"""
url = '{}/{}'.format(self._instance_url, log_file_path)
try:
success, resp = self._make_get_request(url, self._auth_headers)
except SalesforceAppError:
LOGGER.exception('Failed to get event logs')
return
if not (success and resp):
LOGGER.error('Failed to get event logs')
return
# skip header line before passing to rule processor
return resp.splitlines()[1:]
def _generate_headers(self):
"""Each request will request a new token to call the resources APIs.
More details to be found here:
https://developers.onelogin.com/api-docs/1/oauth20-tokens/generate-tokens-2
Returns:
str: Bearer token to be used to call the OneLogin resource APIs
"""
if self._auth_headers:
return True
authorization = 'client_id: {}, client_secret: {}'.format(
self._config.auth['client_id'], self._config.auth['client_secret'])
headers_token = {
'Authorization': authorization,
'Content-Type': 'application/json'
}
result, response = self._make_post_request(
self._token_endpoint(), headers_token,
{'grant_type': 'client_credentials'})
if not result:
return False
if not response:
LOGGER.error(
'Response invalid generating headers for service \'%s\'',
self.type())
return False
bearer = 'bearer:{}'.format(response.get('access_token'))
self._auth_headers = {'Authorization': bearer}
return True
def do_gather():
"""Perform the gather using this scoped method so we can time it"""
logs = self._gather_logs()
# Make sure there are logs, this can be False if there was an issue polling
# of if there are no new logs to be polled
if not logs:
LOGGER.error('Gather process for service \'%s\' was not able to poll any logs',
self.type())
return
# Increment the count of logs gathered
self._gathered_log_count += len(logs)
# Utilize the batcher to send logs to the rule processor
self._batcher.send_logs(self._config['function_name'], logs)
LOGGER.debug('Updating config last timestamp from %d to %d',
self._config.last_timestamp, self._last_timestamp)
# Save the config's last timestamp after each function run
self._config.last_timestamp = self._last_timestamp
self._poll_count += 1
def _validate_status_code(self, resp):
"""Validate status code from get response
Args:
resp (Response): The response object from get request
Returns:
bool: Returns True if status_code is 200. Otherwise returns False,
except
"""
if resp.status_code == 200:
return True
if resp.status_code == 401:
# The OAuth token used has expired or is invalid.
# Retry to renew OAuth token
LOGGER.error(
'The OAuth token used has expired or is invalid. '
'Error code %s, error message %s',
resp.json().get('errorCode', None),
resp.json().get('message', None))
self._request_token()
# Get request will retry when SalesforceAppError exception raised
raise SalesforceAppError
elif resp.status_code == 403 and resp.json().get(
'errorCode') == 'REQUEST_LIMIT_EXCEEDED':
# Exceeded API request limits in your org. Log this information for
# future reference.
LOGGER.error('Exceeded API request limits')
return False
elif resp.status_code == 500:
# Server internal error. Get request will retry.
raise SalesforceAppError
elif resp.status_code > 200:
LOGGER.error(
'Unexpected status code %d detected, error message %s',
resp.status_code, resp.json())
return False
def _get_onelogin_events(self):
"""Get all events from the endpoint for this timeframe
Returns:
[
{
'id': <int:id>,
'created_at': <str:created_at>,
'account_id': <int:account_id>,
'user_id': <int:user_id>,
'event_type_id': <int:event_type_id>,
'notes': <str:notes>,
'ipaddr': <str:ipaddr>,
'actor_user_id': <int:actor_user_id>,
'assuming_acting_user_id': null,
'role_id': <int:role_id>,
'app_id': <int:app_id>,
'group_id': <int:group_id>,
'otp_device_id': <int:otp_device_id>,
'policy_id': <int:policy_id>,
'actor_system': <str:actor_system>,
'custom_message': <str:custom_message>,
'role_name': <str:role_name>,
'app_name': <str:app_name>,
'group_name': <str:group_name>,
'actor_user_name': <str:actor_user_name>,
'user_name': <str:user_name>,
'policy_name': <str:policy_name>,
'otp_device_name': <str:otp_device_name>,
'operation_name': <str:operation_name>,
'directory_sync_run_id': <int:directory_sync_run_id>,
'directory_id': <int:directory_id>,
'resolution': <str:resolution>,
'client_id': <int:client_id>,
'resource_type_id': <int:resource_type_id>,
'error_description': <str:error_description>
}
]
"""
# Make sure we have authentication headers
if not self._auth_headers:
LOGGER.error('No authentication headers for service \'%s\'',
self.type())
return False
# Are we just getting events or getting paginated events?
if self._next_page_url:
params = None
request_url = self._next_page_url
else:
params = {'since': self._last_timestamp}
request_url = self._events_endpoint()
result, response = self._make_get_request(request_url,
self._auth_headers, params)
if not result:
# If we hit the rate limit, update the sleep time
if response.get('status'):
r_status = response.get('status')
if r_status['code'] == 400 and r_status[
'message'] == 'rate_limit_exceeded':
self._set_rate_limit_sleep()
return False
# Fail if response is invalid
if not response:
LOGGER.error('Response is invalid for service \'%s\'', self.type())
return False
# Set pagination link, if there is any
self._next_page_url = response['pagination']['next_link']
self._more_to_poll = bool(self._next_page_url)
# Adjust the last seen event, if the events list is not empty
if not response['data']:
LOGGER.error('Empty list of events for service \'%s\'',
self.type())
return False
self._last_timestamp = response['data'][-1]['created_at']
# Return the list of events to the caller so they can be send to the batcher
return response['data']
def _list_log_files(self):
"""Fetch a list of available log files by event types.
An event generates log data in real time. However, log files are generated
the day after an event takes place, during nonpeak hours. Therefore, log
file data is unavailable for at least one day after an event.
Returns:
list: a list of log file location or empty list.
An example of log files response json body:
{
"totalSize": 2,
"done": True,
"records": [
{
"attributes": {
"type": "EventLogFile",
"url": "/services/data/v32.0/sobjects/EventLogFile/0ATD00001bROAQ"
},
"Id": "0ATD000000001bROAQ",
"EventType": "Console",
"LogFile": "/services/data/v32.0/sobjects/EventLogFile/0ATD00001bROAQ/LogFile",
"LogDate": "2014-03-14T00:00:00.000+0000",
"LogFileLength": 2692.0
},
{
"attributes": {
"type": "EventLogFile",
"url": "/services/data/v32.0/sobjects/EventLogFile/0ATD000000001SdOAI"
},
"Id": "0ATD000000001SdOAI",
"EventType": "Console",
"LogFile": "/services/data/v32.0/sobjects/EventLogFile/0ATD00001SdOAI/LogFile",
"LogDate": "2014-03-13T00:00:00.000+0000",
"LogFileLength": 1345.0
}
]
}
"""
url = self._SALESFORCE_QUERY_URL.format(
instance_url=self._instance_url,
api_version=self._latest_api_version,
query=self._SALESFORCE_QUERY_FILTERS,
start_time=self._SALESFORCE_CREATE_AFTER.format(
self._last_timestamp),
event_type='AND+EventType+=+\'{}\''.format(self._type()))
success, response = self._make_get_request(url, self._auth_headers)
if not success:
LOGGER.error('Failed to get a list of log files.')
return
log_files = []
if response.get('records'):
log_files.extend([
record['LogFile'] for record in response['records']
if record.get('LogFile')
])
LOGGER.debug('Retrived %d log files', len(log_files))
return log_files
