def _connect(self, next_url):
"""Send API call to ThreatStream with next token and return parsed IOCs
The API call has retry logic up to 3 times.
Args:
next_url (str): url of next token to retrieve more objects from
ThreatStream
Returns:
(tuple): (list, str, bool)
- First object is a list of intelligence.
- Second object is a string of next token to retrieve more IOCs.
- Third object is bool to indicated if retrieve more IOCs from
threat feed.
Return False if next token is empty or threshold of number
of IOCs is reached.
"""
continue_invoke = False
intelligence = list()
https_req = requests.get('{}{}'.format(self._API_URL, next_url),
timeout=10)
if https_req.status_code == 200:
data = https_req.json()
if data.get('objects'):
intelligence.extend(self._process_data(data['objects']))
LOGGER.info('IOC Offset: %d', data['meta']['offset'])
if not (data['meta']['next']
and data['meta']['offset'] < self.threshold):
LOGGER.debug(
'Either next token is empty or IOC offset '
'reaches threshold %d. Stop retrieve more '
'IOCs.', self.threshold)
continue_invoke = False
else:
next_url = data['meta']['next']
continue_invoke = True
elif https_req.status_code == 401:
raise ThreatStreamRequestsError(
'Response status code 401, unauthorized.')
elif https_req.status_code == 500:
raise ThreatStreamRequestsError(
'Response status code 500, retry now.')
else:
raise ThreatStreamRequestsError('Unknown status code {}, '
'do not retry.'.format(
https_req.status_code))
return (intelligence, next_url, continue_invoke)
def handler(event, context):
"""Lambda handler"""
config = load_config()
config.update(parse_lambda_func_arn(context))
threat_stream = ThreatStream(config)
intelligence, next_url, continue_invoke = threat_stream.runner(event)
if intelligence:
LOGGER.info('Write %d IOCs to DynamoDB table', len(intelligence))
threat_stream.write_to_dynamodb_table(intelligence)
if context.get_remaining_time_in_millis() > END_TIME_BUFFER * 1000 and continue_invoke:
invoke_lambda_function(next_url, config)
LOGGER.debug("Time remaining (MS): %s", context.get_remaining_time_in_millis())
def _finalize(self, intel, next_url):
"""Finalize the execution
Send data to dynamo and continue the invocation if necessary.
Arguments:
intel (list): List of intelligence to send to DynamoDB
next_url (str): Next token to retrieve more IOCs
continue_invoke (bool): Whether to retrieve more IOCs from
threat feed. False if next token is empty or threshold of number
of IOCs is reached.
"""
if intel:
LOGGER.info('Write %d IOCs to DynamoDB table', len(intel))
self._write_to_dynamodb_table(intel)
if next_url and self.timing_func() > self._END_TIME_BUFFER * 1000:
self._invoke_lambda_function(next_url)
LOGGER.debug("Time remaining (MS): %s", self.timing_func())
def _connect(self, next_url):
"""Send API call to ThreatStream with next token and return parsed IOCs
The API call has retry logic up to 3 times.
Args:
next_url (str): url of next token to retrieve more objects from
ThreatStream
"""
intelligence = list()
https_req = requests.get('{}{}'.format(self._API_URL, next_url),
timeout=10)
next_url = None
if https_req.status_code == 200:
data = https_req.json()
if data.get('objects'):
intelligence.extend(self._process_data(data['objects']))
LOGGER.info('IOC Offset: %d', data['meta']['offset'])
if not (data['meta']['next']
and data['meta']['offset'] < self.threshold):
LOGGER.debug(
'Either next token is empty or IOC offset reaches threshold '
'%d. Stop retrieve more IOCs.', self.threshold)
else:
next_url = data['meta']['next']
elif https_req.status_code == 401:
raise ThreatStreamRequestsError(
'Response status code 401, unauthorized.')
elif https_req.status_code == 500:
raise ThreatStreamRequestsError(
'Response status code 500, retry now.')
else:
raise ThreatStreamRequestsError(
'Unknown status code {}, do not retry.'.format(
https_req.status_code))
self._finalize(intelligence, next_url)
