def _load_api_creds(self):
"""Retrieve ThreatStream API credentials from Parameter Store"""
if self.api_user and self.api_key:
return # credentials already loaded from SSM
try:
ssm = boto3.client('ssm', self.region)
response = ssm.get_parameter(Name=self.CRED_PARAMETER_NAME,
WithDecryption=True)
except ClientError:
LOGGER.exception('Failed to get SSM parameters')
raise
if not response:
raise ThreatStreamCredsError('Invalid response')
try:
decoded_creds = json.loads(response['Parameter']['Value'])
except ValueError:
raise ThreatStreamCredsError(
'Cannot load value for parameter with name '
'\'{}\'. The value is not valid json: '
'\'{}\''.format(response['Parameter']['Name'],
response['Parameter']['Value']))
self.api_user = decoded_creds['api_user']
self.api_key = decoded_creds['api_key']
if not (self.api_user and self.api_key):
raise ThreatStreamCredsError('API Creds Error')
def _invoke_lambda_function(self, next_url):
"""Invoke lambda function itself with next token to continually retrieve IOCs"""
LOGGER.debug('This invocation is invoked by lambda function self.')
lambda_client = boto3.client('lambda', region_name=self.region)
try:
lambda_client.invoke(FunctionName=self._config['function_name'],
InvocationType='Event',
Payload=json.dumps({'next_url': next_url}),
Qualifier=self._config['qualifier'])
except ClientError as err:
raise ThreatStreamLambdaInvokeError(
'Error invoking function: {}'.format(err))
def invoke_lambda_function(next_url, config):
"""Invoke lambda function itself with next token to continually retrieve IOCs"""
LOGGER.debug('This invoacation is invoked by lambda function self.')
try:
lambda_client = boto3.client('lambda', region_name=config['region'])
lambda_client.invoke(FunctionName=config['function_name'],
InvocationType='Event',
Payload=json.dumps({'next_url': next_url}),
Qualifier=config['qualifier'])
except ClientError as err:
LOGGER.error(
'Lambda client error: %s when lambda function invoke self', err)
raise ThreatStreamLambdaInvokeError
def _connect(self, next_url):
"""Send API call to ThreatStream with next token and return parsed IOCs
The API call has retry logic up to 3 times.
Args:
next_url (str): url of next token to retrieve more objects from
ThreatStream
Returns:
(tuple): (list, str, bool)
- First object is a list of intelligence.
- Second object is a string of next token to retrieve more IOCs.
- Third object is bool to indicated if retrieve more IOCs from
threat feed.
Return False if next token is empty or threshold of number
of IOCs is reached.
"""
continue_invoke = False
intelligence = list()
https_req = requests.get('{}{}'.format(self._API_URL, next_url),
timeout=10)
if https_req.status_code == 200:
data = https_req.json()
if data.get('objects'):
intelligence.extend(self._process_data(data['objects']))
LOGGER.info('IOC Offset: %d', data['meta']['offset'])
if not (data['meta']['next']
and data['meta']['offset'] < self.threshold):
LOGGER.debug(
'Either next token is empty or IOC offset '
'reaches threshold %d. Stop retrieve more '
'IOCs.', self.threshold)
continue_invoke = False
else:
next_url = data['meta']['next']
continue_invoke = True
elif https_req.status_code == 401:
raise ThreatStreamRequestsError(
'Response status code 401, unauthorized.')
elif https_req.status_code == 500:
raise ThreatStreamRequestsError(
'Response status code 500, retry now.')
else:
raise ThreatStreamRequestsError('Unknown status code {}, '
'do not retry.'.format(
https_req.status_code))
return (intelligence, next_url, continue_invoke)
def handler(event, context):
"""Lambda handler"""
config = load_config()
config.update(parse_lambda_func_arn(context))
threat_stream = ThreatStream(config)
intelligence, next_url, continue_invoke = threat_stream.runner(event)
if intelligence:
LOGGER.info('Write %d IOCs to DynamoDB table', len(intelligence))
threat_stream.write_to_dynamodb_table(intelligence)
if context.get_remaining_time_in_millis() > END_TIME_BUFFER * 1000 and continue_invoke:
invoke_lambda_function(next_url, config)
LOGGER.debug("Time remaining (MS): %s", context.get_remaining_time_in_millis())
def write_to_dynamodb_table(self, intelligence):
"""Store IOCs to DynamoDB table"""
try:
dynamodb = boto3.resource('dynamodb', region_name=self.region)
table = dynamodb.Table(self.table_name)
with table.batch_writer() as batch:
for ioc in intelligence:
batch.put_item(
Item={
'value': ioc['value'],
'type': ioc['type'],
'sub_type': ioc['itype'],
'source': ioc['source'],
'expiration_ts': ioc['expiration_ts']
})
except ClientError as err:
LOGGER.debug('DynamoDB client error: %s', err)
raise
def _finalize(self, intel, next_url):
"""Finalize the execution
Send data to dynamo and continue the invocation if necessary.
Arguments:
intel (list): List of intelligence to send to DynamoDB
next_url (str): Next token to retrieve more IOCs
continue_invoke (bool): Whether to retrieve more IOCs from
threat feed. False if next token is empty or threshold of number
of IOCs is reached.
"""
if intel:
LOGGER.info('Write %d IOCs to DynamoDB table', len(intel))
self._write_to_dynamodb_table(intel)
if next_url and self.timing_func() > self._END_TIME_BUFFER * 1000:
self._invoke_lambda_function(next_url)
LOGGER.debug("Time remaining (MS): %s", self.timing_func())
def _get_api_creds(self):
"""Retrieve ThreatStream API credentials from Parameter Store"""
try:
ssm = boto3.client('ssm', self.region)
response = ssm.get_parameters(Names=[self._PARAMETER_NAME],
WithDecryption=True)
except ClientError as err:
LOGGER.error('SSM client error: %s', err)
raise
for cred in response['Parameters']:
if cred['Name'] == self._PARAMETER_NAME:
try:
decoded_creds = json.loads(cred['Value'])
self.api_user = decoded_creds['api_user']
self.api_key = decoded_creds['api_key']
except ValueError:
LOGGER.error(
'Can not load value for parameter with '
'name \'%s\'. The value is not valid json: '
'\'%s\'', cred['Name'], cred['Value'])
raise ThreatStreamCredsError('ValueError')
if not (self.api_user and self.api_key):
LOGGER.error('API Creds Error')
raise ThreatStreamCredsError('API Creds Error')
def _connect(self, next_url):
"""Send API call to ThreatStream with next token and return parsed IOCs
The API call has retry logic up to 3 times.
Args:
next_url (str): url of next token to retrieve more objects from
ThreatStream
"""
intelligence = list()
https_req = requests.get('{}{}'.format(self._API_URL, next_url),
timeout=10)
next_url = None
if https_req.status_code == 200:
data = https_req.json()
if data.get('objects'):
intelligence.extend(self._process_data(data['objects']))
LOGGER.info('IOC Offset: %d', data['meta']['offset'])
if not (data['meta']['next']
and data['meta']['offset'] < self.threshold):
LOGGER.debug(
'Either next token is empty or IOC offset reaches threshold '
'%d. Stop retrieve more IOCs.', self.threshold)
else:
next_url = data['meta']['next']
elif https_req.status_code == 401:
raise ThreatStreamRequestsError(
'Response status code 401, unauthorized.')
elif https_req.status_code == 500:
raise ThreatStreamRequestsError(
'Response status code 500, retry now.')
else:
raise ThreatStreamRequestsError(
'Unknown status code {}, do not retry.'.format(
https_req.status_code))
self._finalize(intelligence, next_url)
def _epoch_time(time_str, days=90):
"""Convert expiration time (in UTC) to epoch time
Args:
time_str (str): expiration time in string format
Example: '2017-12-19T04:45:18.412Z'
days (int): default expiration days which 90 days from now
Returns:
(int): Epoch time. If no expiration time presented, return to
default value which is current time + 90 days.
"""
if not time_str:
return int((datetime.utcnow() + timedelta(days) -
datetime.utcfromtimestamp(0)).total_seconds())
try:
utc_time = datetime.strptime(time_str, "%Y-%m-%dT%H:%M:%S.%fZ")
return int(
(utc_time - datetime.utcfromtimestamp(0)).total_seconds())
except ValueError:
LOGGER.error('Cannot convert expiration date \'%s\' to epoch time',
time_str)
raise
