def _load_api_creds(self): """Retrieve ThreatStream API credentials from Parameter Store""" if self.api_user and self.api_key: return # credentials already loaded from SSM try: ssm = boto3.client('ssm', self.region) response = ssm.get_parameter(Name=self.CRED_PARAMETER_NAME, WithDecryption=True) except ClientError: LOGGER.exception('Failed to get SSM parameters') raise if not response: raise ThreatStreamCredsError('Invalid response') try: decoded_creds = json.loads(response['Parameter']['Value']) except ValueError: raise ThreatStreamCredsError( 'Cannot load value for parameter with name ' '\'{}\'. The value is not valid json: ' '\'{}\''.format(response['Parameter']['Name'], response['Parameter']['Value'])) self.api_user = decoded_creds['api_user'] self.api_key = decoded_creds['api_key'] if not (self.api_user and self.api_key): raise ThreatStreamCredsError('API Creds Error')
def _invoke_lambda_function(self, next_url): """Invoke lambda function itself with next token to continually retrieve IOCs""" LOGGER.debug('This invocation is invoked by lambda function self.') lambda_client = boto3.client('lambda', region_name=self.region) try: lambda_client.invoke(FunctionName=self._config['function_name'], InvocationType='Event', Payload=json.dumps({'next_url': next_url}), Qualifier=self._config['qualifier']) except ClientError as err: raise ThreatStreamLambdaInvokeError( 'Error invoking function: {}'.format(err))
def invoke_lambda_function(next_url, config): """Invoke lambda function itself with next token to continually retrieve IOCs""" LOGGER.debug('This invoacation is invoked by lambda function self.') try: lambda_client = boto3.client('lambda', region_name=config['region']) lambda_client.invoke(FunctionName=config['function_name'], InvocationType='Event', Payload=json.dumps({'next_url': next_url}), Qualifier=config['qualifier']) except ClientError as err: LOGGER.error( 'Lambda client error: %s when lambda function invoke self', err) raise ThreatStreamLambdaInvokeError
def _connect(self, next_url): """Send API call to ThreatStream with next token and return parsed IOCs The API call has retry logic up to 3 times. Args: next_url (str): url of next token to retrieve more objects from ThreatStream Returns: (tuple): (list, str, bool) - First object is a list of intelligence. - Second object is a string of next token to retrieve more IOCs. - Third object is bool to indicated if retrieve more IOCs from threat feed. Return False if next token is empty or threshold of number of IOCs is reached. """ continue_invoke = False intelligence = list() https_req = requests.get('{}{}'.format(self._API_URL, next_url), timeout=10) if https_req.status_code == 200: data = https_req.json() if data.get('objects'): intelligence.extend(self._process_data(data['objects'])) LOGGER.info('IOC Offset: %d', data['meta']['offset']) if not (data['meta']['next'] and data['meta']['offset'] < self.threshold): LOGGER.debug( 'Either next token is empty or IOC offset ' 'reaches threshold %d. Stop retrieve more ' 'IOCs.', self.threshold) continue_invoke = False else: next_url = data['meta']['next'] continue_invoke = True elif https_req.status_code == 401: raise ThreatStreamRequestsError( 'Response status code 401, unauthorized.') elif https_req.status_code == 500: raise ThreatStreamRequestsError( 'Response status code 500, retry now.') else: raise ThreatStreamRequestsError('Unknown status code {}, ' 'do not retry.'.format( https_req.status_code)) return (intelligence, next_url, continue_invoke)
def handler(event, context): """Lambda handler""" config = load_config() config.update(parse_lambda_func_arn(context)) threat_stream = ThreatStream(config) intelligence, next_url, continue_invoke = threat_stream.runner(event) if intelligence: LOGGER.info('Write %d IOCs to DynamoDB table', len(intelligence)) threat_stream.write_to_dynamodb_table(intelligence) if context.get_remaining_time_in_millis() > END_TIME_BUFFER * 1000 and continue_invoke: invoke_lambda_function(next_url, config) LOGGER.debug("Time remaining (MS): %s", context.get_remaining_time_in_millis())
def write_to_dynamodb_table(self, intelligence): """Store IOCs to DynamoDB table""" try: dynamodb = boto3.resource('dynamodb', region_name=self.region) table = dynamodb.Table(self.table_name) with table.batch_writer() as batch: for ioc in intelligence: batch.put_item( Item={ 'value': ioc['value'], 'type': ioc['type'], 'sub_type': ioc['itype'], 'source': ioc['source'], 'expiration_ts': ioc['expiration_ts'] }) except ClientError as err: LOGGER.debug('DynamoDB client error: %s', err) raise
def _finalize(self, intel, next_url): """Finalize the execution Send data to dynamo and continue the invocation if necessary. Arguments: intel (list): List of intelligence to send to DynamoDB next_url (str): Next token to retrieve more IOCs continue_invoke (bool): Whether to retrieve more IOCs from threat feed. False if next token is empty or threshold of number of IOCs is reached. """ if intel: LOGGER.info('Write %d IOCs to DynamoDB table', len(intel)) self._write_to_dynamodb_table(intel) if next_url and self.timing_func() > self._END_TIME_BUFFER * 1000: self._invoke_lambda_function(next_url) LOGGER.debug("Time remaining (MS): %s", self.timing_func())
def _get_api_creds(self): """Retrieve ThreatStream API credentials from Parameter Store""" try: ssm = boto3.client('ssm', self.region) response = ssm.get_parameters(Names=[self._PARAMETER_NAME], WithDecryption=True) except ClientError as err: LOGGER.error('SSM client error: %s', err) raise for cred in response['Parameters']: if cred['Name'] == self._PARAMETER_NAME: try: decoded_creds = json.loads(cred['Value']) self.api_user = decoded_creds['api_user'] self.api_key = decoded_creds['api_key'] except ValueError: LOGGER.error( 'Can not load value for parameter with ' 'name \'%s\'. The value is not valid json: ' '\'%s\'', cred['Name'], cred['Value']) raise ThreatStreamCredsError('ValueError') if not (self.api_user and self.api_key): LOGGER.error('API Creds Error') raise ThreatStreamCredsError('API Creds Error')
def _connect(self, next_url): """Send API call to ThreatStream with next token and return parsed IOCs The API call has retry logic up to 3 times. Args: next_url (str): url of next token to retrieve more objects from ThreatStream """ intelligence = list() https_req = requests.get('{}{}'.format(self._API_URL, next_url), timeout=10) next_url = None if https_req.status_code == 200: data = https_req.json() if data.get('objects'): intelligence.extend(self._process_data(data['objects'])) LOGGER.info('IOC Offset: %d', data['meta']['offset']) if not (data['meta']['next'] and data['meta']['offset'] < self.threshold): LOGGER.debug( 'Either next token is empty or IOC offset reaches threshold ' '%d. Stop retrieve more IOCs.', self.threshold) else: next_url = data['meta']['next'] elif https_req.status_code == 401: raise ThreatStreamRequestsError( 'Response status code 401, unauthorized.') elif https_req.status_code == 500: raise ThreatStreamRequestsError( 'Response status code 500, retry now.') else: raise ThreatStreamRequestsError( 'Unknown status code {}, do not retry.'.format( https_req.status_code)) self._finalize(intelligence, next_url)
def _epoch_time(time_str, days=90): """Convert expiration time (in UTC) to epoch time Args: time_str (str): expiration time in string format Example: '2017-12-19T04:45:18.412Z' days (int): default expiration days which 90 days from now Returns: (int): Epoch time. If no expiration time presented, return to default value which is current time + 90 days. """ if not time_str: return int((datetime.utcnow() + timedelta(days) - datetime.utcfromtimestamp(0)).total_seconds()) try: utc_time = datetime.strptime(time_str, "%Y-%m-%dT%H:%M:%S.%fZ") return int( (utc_time - datetime.utcfromtimestamp(0)).total_seconds()) except ValueError: LOGGER.error('Cannot convert expiration date \'%s\' to epoch time', time_str) raise