def riot(self, ip_address): """Check if IP is in RIOT data set :param ip_address: IP address to use in the look-up. :type ip_address: str :return: Context for the IP address. :rtype: dict """ if self.offering == "community": response = { "message": "RIOT lookup not supported with Community offering" } return response else: LOGGER.debug("Checking RIOT for %s...", ip_address, ip_address=ip_address) validate_ip(ip_address) endpoint = self.EP_RIOT.format(ip_address=ip_address) response = self._request(endpoint) if "ip" not in response: response["ip"] = ip_address return response
def ip(self, ip_address): """Get context associated with an IP address. :param ip_address: IP address to use in the look-up. :type ip_address: str :return: Context for the IP address. :rtype: dict """ LOGGER.debug("Getting context for %s...", ip_address, ip_address=ip_address) validate_ip(ip_address) if self.offering.lower() == "community": endpoint = self.EP_COMMUNITY_IP.format(ip_address=ip_address) else: endpoint = self.EP_NOISE_CONTEXT.format(ip_address=ip_address) if self.use_cache: cache = self.ip_context_cache response = ( cache[ip_address] if ip_address in self.ip_context_cache else cache.setdefault(ip_address, self._request(endpoint)) ) else: response = self._request(endpoint) if "ip" not in response: response["ip"] = ip_address return response
def ip(self, ip_address): """Get context associated with an IP address. :param ip_address: IP address to use in the look-up. :type recurse: str :return: Context for the IP address. :rtype: dict """ LOGGER.debug("Getting context for %s...", ip_address) validate_ip(ip_address) endpoint = self.EP_NOISE_CONTEXT.format(ip_address=ip_address) if self.use_cache: cache = self.IP_CONTEXT_CACHE response = (cache[ip_address] if ip_address in self.IP_CONTEXT_CACHE else cache.setdefault( ip_address, self._request(endpoint))) else: response = self._request(endpoint) if "ip" not in response: response["ip"] = ip_address return response
def event_filter(chunk_index, result, records_dict, ip_field, noise_events, method): """Method for filtering the events based on the noise status.""" api_results = result['response'] error_flag = True # Before yielding events, make the ip lookup dict which will have the following format: # {<ip-address>: <API response for that IP address>} ip_lookup = {} if result['message'] == 'ok': error_flag = False for event in api_results: ip_lookup[event['ip']] = event for record in records_dict[0]: if error_flag: # Exception has occured while fetching the noise statuses from API if ip_field in record and record[ip_field] != '': # These calls have been failed due to API failure, # as this event have IP address value, considering them as noise if noise_events: event = { 'ip': record[ip_field], 'error': api_results } yield event_generator.make_invalid_event(method, event, True, record) else: # Either the record is not having IP field or the value of the IP field is '' # send the record as it is as it doesn't have any IP address, after appending all fields # Considering this event as non-noisy if not noise_events: yield event_generator.make_invalid_event(method, {}, True, record) else: # Successful execution of the API call if ip_field in record and record[ip_field] != '': # Check if the IP field is not an iterable to avoid any error while referencing ip in ip_lookup if isinstance(record[ip_field], six.string_types) and record[ip_field] in ip_lookup: if ip_lookup[record[ip_field]]['noise'] == noise_events: yield event_generator.make_valid_event(method, ip_lookup[record[ip_field]], True, record) else: # Meaning ip is either invalid or not returned by the API, which is case of `multi` method only # Invalid IPs are considered as non-noise if not noise_events: try: validate_ip(record[ip_field], strict=True) except ValueError as ve: error_msg = str(ve).split(":") event = { 'ip': record[ip_field], 'error': error_msg[0] } yield event_generator.make_invalid_event(method, event, True, record) else: if not noise_events: # Either the record is not having IP field or the value of the IP field is '' # send the record as it is as it doesn't have any IP address, after appending all fields # Considering this event as non-noisy yield event_generator.make_invalid_event(method, {}, True, record)
def interesting(self, ip_address): """Report an IP as "interesting". :param ip_address: IP address to report as "interesting". :type ip_address: str """ LOGGER.debug("Reporting interesting IP: %s...", ip_address, ip_address=ip_address) validate_ip(ip_address) endpoint = self.EP_INTERESTING.format(ip_address=ip_address) response = self._request(endpoint, method="post") return response
def ip_addresses_parameter(_context, _parameter, values): """IPv4 addresses passed from the command line. :param values: IPv4 address values :type value: list :raises click.BadParameter: when any IP address value is invalid """ for value in values: try: validate_ip(value) except ValueError: raise click.BadParameter(value) return values
def quick(context, api_client, api_key, input_file, output_format, ip_address): """Quickly check whether or not one or many IPs are "noise".""" if input_file is None and not sys.stdin.isatty(): input_file = sys.stdin if input_file is None and not ip_address: click.echo(context.get_help()) context.exit(-1) ip_addresses = [] if input_file is not None: lines = [line.strip() for line in input_file] ip_addresses.extend( [line for line in lines if validate_ip(line, strict=False)]) ip_addresses.extend(list(ip_address)) if not ip_addresses: output = [ context.command.get_usage(context), ("Error: at least one valid IP address must be passed either as an " "argument (IP_ADDRESS) or through the -i/--input_file option."), ] click.echo("\n\n".join(output)) context.exit(-1) results = [] if ip_addresses: results.extend(api_client.quick(ip_addresses=ip_addresses)) return results
def get_ip_addresses(context, input_file, ip_address): """Get IP addresses passed as argument or via input file. :param context: Subcommand context :type context: click.Context :param input_file: Input file :type input_file: click.File | None :param ip_address: IP addresses passed via the ip address argument :type query: tuple(str, ...) """ if input_file is None and not sys.stdin.isatty(): input_file = click.open_file("-") if input_file is None and not ip_address: click.echo(context.get_help()) context.exit(-1) ip_addresses = [] if input_file is not None: lines = [line.strip() for line in input_file] ip_addresses.extend( [line for line in lines if validate_ip(line, strict=False)]) ip_addresses.extend(list(ip_address)) if not ip_addresses: output = [ context.command.get_usage(context), ("Error: at least one valid IP address must be passed either as an " "argument (IP_ADDRESS) or through the -i/--input_file option."), ] click.echo("\n\n".join(output)) context.exit(-1) return ip_addresses
def ip(context, api_client, api_key, input_file, output_format, verbose, ip_address): """Query GreyNoise for all information on a given IP.""" if input_file is None and not sys.stdin.isatty(): input_file = click.open_file("-") if input_file is None and not ip_address: click.echo(context.get_help()) context.exit(-1) ip_addresses = [] if input_file is not None: lines = [line.strip() for line in input_file] ip_addresses.extend( [line for line in lines if validate_ip(line, strict=False)]) if ip_address: ip_addresses.append(ip_address) if not ip_addresses: output = [ context.command.get_usage(context), ("Error: at least one valid IP address must be passed either as an " "argument (IP_ADDRESS) or through the -i/--input_file option."), ] click.echo("\n\n".join(output)) context.exit(-1) results = [ api_client.ip(ip_address=ip_address) for ip_address in ip_addresses ] return results
def ip_address_parameter(_context, _parameter, value): """IPv4 address passed from the command line. :param value: IPv4 address value :type value: str :raises click.BadParameter: when IP address value is invalid """ if value is None: return value try: validate_ip(value) except ValueError: raise click.BadParameter(value) return value
def interesting(self, ip_address): """Report an IP as "interesting". :param ip_address: IP address to report as "interesting". :type ip_address: str """ if self.offering == "community": response = { "message": "Interesting report not supported with Community offering" } return response else: LOGGER.debug("Reporting interesting IP: %s...", ip_address, ip_address=ip_address) validate_ip(ip_address) endpoint = self.EP_INTERESTING.format(ip_address=ip_address) response = self._request(endpoint, method="post") return response
def transform(self, records): """Method that processes and yield event records to the Splunk events pipeline.""" ip_addresses = self.ip ip_field = self.ip_field api_key = "" EVENTS_PER_CHUNK = 5000 THREADS = 3 USE_CACHE = False logger = utility.setup_logger( session_key=self._metadata.searchinfo.session_key, log_context=self._metadata.searchinfo.command) if ip_addresses and ip_field: logger.error( "Please use parameter ip to work gnquick as generating command or " "use parameter ip_field to work gnquick as transforming command." ) self.write_error( "Please use parameter ip to work gnquick as generating command or " "use parameter ip_field to work gnquick as transforming command" ) exit(1) try: message = '' api_key = utility.get_api_key( self._metadata.searchinfo.session_key, logger=logger) except APIKeyNotFoundError as e: message = str(e) except HTTPError as e: message = str(e) if message: self.write_error(message) logger.error( "Error occured while retrieving API key, Error: {}".format( message)) exit(1) if ip_addresses and not ip_field: # This peice of code will work as generating command and will not use the Splunk events. # Splitting the ip_addresses by commas and stripping spaces from both the sides for each IP address ip_addresses = [ip.strip() for ip in ip_addresses.split(',')] logger.info("Started retrieving results") try: logger.debug( "Initiating to fetch noise and RIOT status for IP address(es): {}" .format(str(ip_addresses))) api_client = GreyNoise(api_key=api_key, timeout=120, integration_name=INTEGRATION_NAME) # CACHING START cache_enabled, cache_client = utility.get_caching( self._metadata.searchinfo.session_key, 'multi', logger) if int(cache_enabled) == 1 and cache_client is not None: cache_start = time.time() ips_not_in_cache, ips_in_cache = utility.get_ips_not_in_cache( cache_client, ip_addresses, logger) try: response = [] if len(ips_in_cache) >= 1: response = cache_client.query_kv_store( ips_in_cache) if response is None: logger.debug( "KVStore is not ready. Skipping caching mechanism." ) noise_status = api_client.quick(ip_addresses) elif response == []: noise_status = utility.fetch_response_from_api( api_client.quick, cache_client, ip_addresses, logger) else: noise_status = utility.fetch_response_from_api( api_client.quick, cache_client, ips_not_in_cache, logger) noise_status.extend(response) except Exception: logger.debug( "An exception occurred while fetching response from cache.\n{}" .format(traceback.format_exc())) logger.debug( "Generating command with caching took {} seconds.". format(time.time() - cache_start)) else: # Opting timout 120 seconds for the requests noise_status = api_client.quick(ip_addresses) logger.info("Retrieved results successfully") # CACHING END # Process the API response and send the noise and RIOT status information of IP with extractions # to the Splunk, Using this flag to handle the field extraction issue in custom commands # Only the fields extracted from the first event of generated by custom command # will be extracted from all events first_record_flag = True # Flag to indicate whether erroneous IPs are present erroneous_ip_present = False for ip in ip_addresses: for sample in noise_status: if ip == sample['ip']: yield event_generator.make_valid_event( 'quick', sample, first_record_flag) if first_record_flag: first_record_flag = False logger.debug( "Fetched noise and RIOT status for ip={} from GreyNoise API" .format(str(ip))) break else: erroneous_ip_present = True try: validate_ip(ip, strict=True) except ValueError as e: error_msg = str(e).split(":") logger.debug( "Generating noise and RIOT status for ip={} manually" .format(str(ip))) event = {'ip': ip, 'error': error_msg[0]} yield event_generator.make_invalid_event( 'quick', event, first_record_flag) if first_record_flag: first_record_flag = False if erroneous_ip_present: logger.warn( "Value of one or more IP address(es) is either invalid or non-routable" ) self.write_warning( "Value of one or more IP address(es) passed to {command_name} " "is either invalid or non-routable".format( command_name=str( self._metadata.searchinfo.command))) except RateLimitError: logger.error( "Rate limit error occured while fetching the context information for ips={}" .format(str(ip_addresses))) self.write_error( "The Rate Limit has been exceeded. Please contact the Administrator" ) except RequestFailure as e: response_code, response_message = e.args if response_code == 401: msg = "Unauthorized. Please check your API key." else: # Need to handle this, as splunklib is unable to handle the exception with # (400, {'error': 'error_reason'}) format msg = ( "The API call to the GreyNoise platform have been failed " "with status_code: {} and error: {}").format( response_code, response_message['error'] if isinstance( response_message, dict) else response_message) logger.error("{}".format(str(msg))) self.write_error(msg) except ConnectionError: logger.error( "Error while connecting to the Server. Please check your connection and try again." ) self.write_error( "Error while connecting to the Server. Please check your connection and try again." ) except RequestException: logger.error( "There was an ambiguous exception that occurred while handling your Request. Please try again." ) self.write_error( "There was an ambiguous exception that occurred while handling your Request. Please try again." ) except Exception: logger.error("Exception: {} ".format( str(traceback.format_exc()))) self.write_error( "Exception occured while fetching the noise and RIOT status of the IP address(es). " "See greynoise_main.log for more details.") elif ip_field: # Enter the mechanism only when the Search is complete and all the events are available if self.search_results_info and not self.metadata.preview: try: # Strip the spaces from the parameter value if given ip_field = ip_field.strip() # Validating the given parameter try: ip_field = validator.Fieldname( option_name='ip_field').validate(ip_field) except ValueError as e: # Validator will throw ValueError with error message when the parameters are not proper logger.error(str(e)) self.write_error(str(e)) exit(1) # API key validation if not self.api_validation_flag: api_key_validation, message = utility.validate_api_key( api_key, logger) logger.debug( "API validation status: {}, message: {}".format( api_key_validation, str(message))) self.api_validation_flag = True if not api_key_validation: logger.info(message) self.write_error(message) exit(1) # This piece of code will work as transforming command and will use # the Splunk ingested events and field which is specified in ip_field. chunk_dict = event_generator.batch(records, ip_field, EVENTS_PER_CHUNK, logger) # This means there are only 1000 or below IPs to call in the entire bunch of records # Use one thread with single thread with caching mechanism enabled for the chunk if len(chunk_dict) == 1: logger.info( "Less then 1000 distinct IPs are present, " "optimizing the IP requests call to GreyNoise API..." ) THREADS = 1 USE_CACHE = True api_client = GreyNoise(api_key=api_key, timeout=120, use_cache=USE_CACHE, integration_name=INTEGRATION_NAME) # When no records found, batch will return {0:([],[])} tot_time_start = time.time() if len(list(chunk_dict.values())[0][0]) >= 1: for event in event_generator.get_all_events( self._metadata.searchinfo.session_key, api_client, 'multi', ip_field, chunk_dict, logger, threads=THREADS): yield event else: logger.info( "No events found, please increase the search timespan to have more search results." ) tot_time_end = time.time() logger.debug( "Total execution time => {}".format(tot_time_end - tot_time_start)) except Exception: logger.info( "Exception occured while adding the noise and RIOT status to the events, Error: {}" .format(traceback.format_exc())) self.write_error( "Exception occured while adding the noise and RIOT status of " "the IP addresses to events. See greynoise_main.log for more details." ) else: logger.error( "Please specify exactly one parameter from ip and ip_field with some value." ) self.write_error( "Please specify exactly one parameter from ip and ip_field with some value." )
def event_processor(records_dict, result, method, ip_field, logger): """ Process on each chunk, format response retrieved from API and Send the results of transforming command to Splunk. :param records_dict: Tuple having all the records of the chunk and all the IP addresses present in the ip_field :param method: method used for the API invocation :param ip_field: name of the field representing the IP address in Splunk events :param logger: logger instance :return: dict denoting the event to send to Splunk """ generate_missing_events, result = method_response_mapper( method, result, logger) # Loading the response to avoid loading it each time # This will either have API response for the chunk or # the exception message denoting exception occured while fetching the data if result['response'] != []: if type(result['response'][0]) == list: api_results = [] for each in result['response'][0]: api_results.append(each) else: api_results = result['response'] else: api_results = result['response'] error_flag = True # Before yielding events, make the ip lookup dict which will have the following format: # {<ip-address>: <API response for that IP address>} ip_lookup = {} if result['message'] == 'ok': error_flag = False for event in api_results: ip_lookup[event['ip']] = event # This will be called per chunk to yield the events as per the objective of trasnforming command for record in records_dict[0]: if error_flag: # Exception has occured while fetching the data if ip_field in record and record[ip_field]: event = {'ip': record[ip_field], 'error': api_results} yield make_invalid_event(method, event, True, record) else: # Either the record is not having IP field or the value of the IP field is '' # send the record as it is as it doesn't have any IP address, after appending all fields yield make_invalid_event(method, {}, True, record) else: # Successful execution of the API call if ip_field in record and record[ip_field]: # Check if the IP field is not an iterable to avoid any error while referencing ip in ip_lookup if isinstance( record[ip_field], six.string_types) and record[ip_field] in ip_lookup: # Deleting the raw_data from the response when the request method is enrich if method == 'enrich' and 'raw_data' in ip_lookup[ record[ip_field]]: del ip_lookup[record[ip_field]]['raw_data'] yield make_valid_event(method, ip_lookup[record[ip_field]], True, record) else: # Meaning ip is either invalid or not returned by the API, # happens when quick method is used while retrieving data if generate_missing_events: try: validate_ip(record[ip_field], strict=True) except ValueError as e: error_msg = str(e).split(":") event = { 'ip': record[ip_field], 'error': error_msg[0] } yield make_invalid_event(method, event, True, record) else: # Either the record is not having IP field or the value of the IP field is '' # send the record as it is as it doesn't have any IP address, after appending all fields yield make_invalid_event(method, {}, True, record)
def quick(self, ip_addresses): """Get activity associated with one or more IP addresses. :param ip_addresses: One or more IP addresses to use in the look-up. :type ip_addresses: str | list :return: Bulk status information for IP addresses. :rtype: dict """ LOGGER.debug("Getting noise status for %s...", ip_addresses) if isinstance(ip_addresses, str): ip_addresses = [ip_addresses] ip_addresses = [ ip_address for ip_address in ip_addresses if validate_ip(ip_address, strict=False) ] if self.use_cache: cache = self.IP_QUICK_CHECK_CACHE # Keep the same ordering as in the input ordered_results = OrderedDict((ip_address, cache.get(ip_address)) for ip_address in ip_addresses) api_ip_addresses = [ ip_address for ip_address, result in ordered_results.items() if result is None ] if api_ip_addresses: api_results = [] if len(api_ip_addresses) == 1: endpoint = self.EP_NOISE_QUICK.format( ip_address=api_ip_addresses[0]) api_results.append(self._request(endpoint)) else: for chunk in more_itertools.chunked( api_ip_addresses, self.IP_QUICK_CHECK_CHUNK_SIZE): api_results.extend( self._request(self.EP_NOISE_MULTI, json={"ips": chunk})) for api_result in api_results: ip_address = api_result["ip"] ordered_results[ip_address] = cache.setdefault( ip_address, api_result) results = list(ordered_results.values()) else: results = [] if len(ip_addresses) == 1: endpoint = self.EP_NOISE_QUICK.format( ip_address=ip_addresses[0]) results.append(self._request(endpoint)) else: for chunk in more_itertools.chunked( ip_addresses, self.IP_QUICK_CHECK_CHUNK_SIZE): results.extend( self._request(self.EP_NOISE_MULTI, json={"ips": chunk})) for result in results: code = result["code"] result["code_message"] = self.CODE_MESSAGES.get( code, self.UNKNOWN_CODE_MESSAGE.format(code)) return results
def test_invalid(self, ip): """Invalid ip address values.""" with pytest.raises(ValueError) as exception: validate_ip(ip) assert str(exception.value) == "Invalid IP address: {!r}".format(ip)
def quick(self, ip_addresses, include_invalid=False): # noqa: C901 """Get activity associated with one or more IP addresses. :param ip_addresses: One or more IP addresses to use in the look-up. :type ip_addresses: str | list :return: Bulk status information for IP addresses. :rtype: dict :param include_invalid: True or False :type include_invalid: bool """ if self.offering == "community": response = [ {"message": "Quick Lookup not supported with Community offering"} ] return response else: if isinstance(ip_addresses, str): ip_addresses = ip_addresses.split(",") LOGGER.debug("Getting noise status...", ip_addresses=ip_addresses) valid_ip_addresses = [ ip_address for ip_address in ip_addresses if validate_ip(ip_address, strict=False) ] if self.use_cache: cache = self.ip_quick_check_cache # Keep the same ordering as in the input ordered_results = OrderedDict( (ip_address, cache.get(ip_address)) for ip_address in valid_ip_addresses ) api_ip_addresses = [ ip_address for ip_address, result in ordered_results.items() if result is None ] if api_ip_addresses: api_results = [] chunks = more_itertools.chunked( api_ip_addresses, self.IP_QUICK_CHECK_CHUNK_SIZE ) for chunk in chunks: api_result = self._request( self.EP_NOISE_MULTI, json={"ips": chunk} ) if isinstance(api_result, list): api_results.extend(api_result) else: api_results.append(api_result) for api_result in api_results: ip_address = api_result["ip"] ordered_results[ip_address] = cache.setdefault( ip_address, api_result ) results = list(ordered_results.values()) else: results = [] chunks = more_itertools.chunked( valid_ip_addresses, self.IP_QUICK_CHECK_CHUNK_SIZE ) for chunk in chunks: result = self._request(self.EP_NOISE_MULTI, json={"ips": chunk}) if isinstance(result, list): results.extend(result) else: results.append(result) [ results.append({"ip": ip, "noise": False, "code": "404"}) for ip in ip_addresses if ip not in valid_ip_addresses and include_invalid ] for result in results: code = result["code"] result["code_message"] = self.CODE_MESSAGES.get( code, self.UNKNOWN_CODE_MESSAGE.format(code) ) return results
def test_valid(self, ip): """Valid ip address values.""" validate_ip(ip)