def _invoke_lambda_function(self, next_url): """Invoke lambda function itself with next token to continually retrieve IOCs""" LOGGER.debug('This invocation is invoked by lambda function self.') lambda_client = boto3.client('lambda', region_name=self.region) try: lambda_client.invoke(FunctionName=self._config['function_name'], InvocationType='Event', Payload=json.dumps({'next_url': next_url}), Qualifier=self._config['qualifier']) except ClientError as err: raise ThreatStreamLambdaInvokeError( 'Error invoking function: {}'.format(err))
def invoke_lambda_function(next_url, config): """Invoke lambda function itself with next token to continually retrieve IOCs""" LOGGER.debug('This invoacation is invoked by lambda function self.') try: lambda_client = boto3.client('lambda', region_name=config['region']) lambda_client.invoke(FunctionName=config['function_name'], InvocationType='Event', Payload=json.dumps({'next_url': next_url}), Qualifier=config['qualifier']) except ClientError as err: LOGGER.error( 'Lambda client error: %s when lambda function invoke self', err) raise ThreatStreamLambdaInvokeError
def _connect(self, next_url): """Send API call to ThreatStream with next token and return parsed IOCs The API call has retry logic up to 3 times. Args: next_url (str): url of next token to retrieve more objects from ThreatStream Returns: (tuple): (list, str, bool) - First object is a list of intelligence. - Second object is a string of next token to retrieve more IOCs. - Third object is bool to indicated if retrieve more IOCs from threat feed. Return False if next token is empty or threshold of number of IOCs is reached. """ continue_invoke = False intelligence = list() https_req = requests.get('{}{}'.format(self._API_URL, next_url), timeout=10) if https_req.status_code == 200: data = https_req.json() if data.get('objects'): intelligence.extend(self._process_data(data['objects'])) LOGGER.info('IOC Offset: %d', data['meta']['offset']) if not (data['meta']['next'] and data['meta']['offset'] < self.threshold): LOGGER.debug( 'Either next token is empty or IOC offset ' 'reaches threshold %d. Stop retrieve more ' 'IOCs.', self.threshold) continue_invoke = False else: next_url = data['meta']['next'] continue_invoke = True elif https_req.status_code == 401: raise ThreatStreamRequestsError( 'Response status code 401, unauthorized.') elif https_req.status_code == 500: raise ThreatStreamRequestsError( 'Response status code 500, retry now.') else: raise ThreatStreamRequestsError('Unknown status code {}, ' 'do not retry.'.format( https_req.status_code)) return (intelligence, next_url, continue_invoke)
def handler(event, context): """Lambda handler""" config = load_config() config.update(parse_lambda_func_arn(context)) threat_stream = ThreatStream(config) intelligence, next_url, continue_invoke = threat_stream.runner(event) if intelligence: LOGGER.info('Write %d IOCs to DynamoDB table', len(intelligence)) threat_stream.write_to_dynamodb_table(intelligence) if context.get_remaining_time_in_millis() > END_TIME_BUFFER * 1000 and continue_invoke: invoke_lambda_function(next_url, config) LOGGER.debug("Time remaining (MS): %s", context.get_remaining_time_in_millis())
def write_to_dynamodb_table(self, intelligence): """Store IOCs to DynamoDB table""" try: dynamodb = boto3.resource('dynamodb', region_name=self.region) table = dynamodb.Table(self.table_name) with table.batch_writer() as batch: for ioc in intelligence: batch.put_item( Item={ 'value': ioc['value'], 'type': ioc['type'], 'sub_type': ioc['itype'], 'source': ioc['source'], 'expiration_ts': ioc['expiration_ts'] }) except ClientError as err: LOGGER.debug('DynamoDB client error: %s', err) raise
def _finalize(self, intel, next_url): """Finalize the execution Send data to dynamo and continue the invocation if necessary. Arguments: intel (list): List of intelligence to send to DynamoDB next_url (str): Next token to retrieve more IOCs continue_invoke (bool): Whether to retrieve more IOCs from threat feed. False if next token is empty or threshold of number of IOCs is reached. """ if intel: LOGGER.info('Write %d IOCs to DynamoDB table', len(intel)) self._write_to_dynamodb_table(intel) if next_url and self.timing_func() > self._END_TIME_BUFFER * 1000: self._invoke_lambda_function(next_url) LOGGER.debug("Time remaining (MS): %s", self.timing_func())
def _connect(self, next_url): """Send API call to ThreatStream with next token and return parsed IOCs The API call has retry logic up to 3 times. Args: next_url (str): url of next token to retrieve more objects from ThreatStream """ intelligence = list() https_req = requests.get('{}{}'.format(self._API_URL, next_url), timeout=10) next_url = None if https_req.status_code == 200: data = https_req.json() if data.get('objects'): intelligence.extend(self._process_data(data['objects'])) LOGGER.info('IOC Offset: %d', data['meta']['offset']) if not (data['meta']['next'] and data['meta']['offset'] < self.threshold): LOGGER.debug( 'Either next token is empty or IOC offset reaches threshold ' '%d. Stop retrieve more IOCs.', self.threshold) else: next_url = data['meta']['next'] elif https_req.status_code == 401: raise ThreatStreamRequestsError( 'Response status code 401, unauthorized.') elif https_req.status_code == 500: raise ThreatStreamRequestsError( 'Response status code 500, retry now.') else: raise ThreatStreamRequestsError( 'Unknown status code {}, do not retry.'.format( https_req.status_code)) self._finalize(intelligence, next_url)