def riot(self, ip_address):
"""Check if IP is in RIOT data set
:param ip_address: IP address to use in the look-up.
:type ip_address: str
:return: Context for the IP address.
:rtype: dict
"""
if self.offering == "community":
response = {
"message": "RIOT lookup not supported with Community offering"
}
return response
else:
LOGGER.debug("Checking RIOT for %s...",
ip_address,
ip_address=ip_address)
validate_ip(ip_address)
endpoint = self.EP_RIOT.format(ip_address=ip_address)
response = self._request(endpoint)
if "ip" not in response:
response["ip"] = ip_address
return response
def ip(self, ip_address):
"""Get context associated with an IP address.
:param ip_address: IP address to use in the look-up.
:type ip_address: str
:return: Context for the IP address.
:rtype: dict
"""
LOGGER.debug("Getting context for %s...", ip_address, ip_address=ip_address)
validate_ip(ip_address)
if self.offering.lower() == "community":
endpoint = self.EP_COMMUNITY_IP.format(ip_address=ip_address)
else:
endpoint = self.EP_NOISE_CONTEXT.format(ip_address=ip_address)
if self.use_cache:
cache = self.ip_context_cache
response = (
cache[ip_address]
if ip_address in self.ip_context_cache
else cache.setdefault(ip_address, self._request(endpoint))
)
else:
response = self._request(endpoint)
if "ip" not in response:
response["ip"] = ip_address
return response
def ip(self, ip_address):
"""Get context associated with an IP address.
:param ip_address: IP address to use in the look-up.
:type recurse: str
:return: Context for the IP address.
:rtype: dict
"""
LOGGER.debug("Getting context for %s...", ip_address)
validate_ip(ip_address)
endpoint = self.EP_NOISE_CONTEXT.format(ip_address=ip_address)
if self.use_cache:
cache = self.IP_CONTEXT_CACHE
response = (cache[ip_address] if ip_address
in self.IP_CONTEXT_CACHE else cache.setdefault(
ip_address, self._request(endpoint)))
else:
response = self._request(endpoint)
if "ip" not in response:
response["ip"] = ip_address
return response
def event_filter(chunk_index, result, records_dict, ip_field, noise_events, method):
"""Method for filtering the events based on the noise status."""
api_results = result['response']
error_flag = True
# Before yielding events, make the ip lookup dict which will have the following format:
# {<ip-address>: <API response for that IP address>}
ip_lookup = {}
if result['message'] == 'ok':
error_flag = False
for event in api_results:
ip_lookup[event['ip']] = event
for record in records_dict[0]:
if error_flag:
# Exception has occured while fetching the noise statuses from API
if ip_field in record and record[ip_field] != '':
# These calls have been failed due to API failure,
# as this event have IP address value, considering them as noise
if noise_events:
event = {
'ip': record[ip_field],
'error': api_results
}
yield event_generator.make_invalid_event(method, event, True, record)
else:
# Either the record is not having IP field or the value of the IP field is ''
# send the record as it is as it doesn't have any IP address, after appending all fields
# Considering this event as non-noisy
if not noise_events:
yield event_generator.make_invalid_event(method, {}, True, record)
else:
# Successful execution of the API call
if ip_field in record and record[ip_field] != '':
# Check if the IP field is not an iterable to avoid any error while referencing ip in ip_lookup
if isinstance(record[ip_field], six.string_types) and record[ip_field] in ip_lookup:
if ip_lookup[record[ip_field]]['noise'] == noise_events:
yield event_generator.make_valid_event(method, ip_lookup[record[ip_field]], True, record)
else:
# Meaning ip is either invalid or not returned by the API, which is case of `multi` method only
# Invalid IPs are considered as non-noise
if not noise_events:
try:
validate_ip(record[ip_field], strict=True)
except ValueError as ve:
error_msg = str(ve).split(":")
event = {
'ip': record[ip_field],
'error': error_msg[0]
}
yield event_generator.make_invalid_event(method, event, True, record)
else:
if not noise_events:
# Either the record is not having IP field or the value of the IP field is ''
# send the record as it is as it doesn't have any IP address, after appending all fields
# Considering this event as non-noisy
yield event_generator.make_invalid_event(method, {}, True, record)
def interesting(self, ip_address):
"""Report an IP as "interesting".
:param ip_address: IP address to report as "interesting".
:type ip_address: str
"""
LOGGER.debug("Reporting interesting IP: %s...",
ip_address,
ip_address=ip_address)
validate_ip(ip_address)
endpoint = self.EP_INTERESTING.format(ip_address=ip_address)
response = self._request(endpoint, method="post")
return response
def ip_addresses_parameter(_context, _parameter, values):
"""IPv4 addresses passed from the command line.
:param values: IPv4 address values
:type value: list
:raises click.BadParameter: when any IP address value is invalid
"""
for value in values:
try:
validate_ip(value)
except ValueError:
raise click.BadParameter(value)
return values
def quick(context, api_client, api_key, input_file, output_format, ip_address):
"""Quickly check whether or not one or many IPs are "noise"."""
if input_file is None and not sys.stdin.isatty():
input_file = sys.stdin
if input_file is None and not ip_address:
click.echo(context.get_help())
context.exit(-1)
ip_addresses = []
if input_file is not None:
lines = [line.strip() for line in input_file]
ip_addresses.extend(
[line for line in lines if validate_ip(line, strict=False)])
ip_addresses.extend(list(ip_address))
if not ip_addresses:
output = [
context.command.get_usage(context),
("Error: at least one valid IP address must be passed either as an "
"argument (IP_ADDRESS) or through the -i/--input_file option."),
]
click.echo("\n\n".join(output))
context.exit(-1)
results = []
if ip_addresses:
results.extend(api_client.quick(ip_addresses=ip_addresses))
return results
def get_ip_addresses(context, input_file, ip_address):
"""Get IP addresses passed as argument or via input file.
:param context: Subcommand context
:type context: click.Context
:param input_file: Input file
:type input_file: click.File | None
:param ip_address: IP addresses passed via the ip address argument
:type query: tuple(str, ...)
"""
if input_file is None and not sys.stdin.isatty():
input_file = click.open_file("-")
if input_file is None and not ip_address:
click.echo(context.get_help())
context.exit(-1)
ip_addresses = []
if input_file is not None:
lines = [line.strip() for line in input_file]
ip_addresses.extend(
[line for line in lines if validate_ip(line, strict=False)])
ip_addresses.extend(list(ip_address))
if not ip_addresses:
output = [
context.command.get_usage(context),
("Error: at least one valid IP address must be passed either as an "
"argument (IP_ADDRESS) or through the -i/--input_file option."),
]
click.echo("\n\n".join(output))
context.exit(-1)
return ip_addresses
def ip(context, api_client, api_key, input_file, output_format, verbose,
ip_address):
"""Query GreyNoise for all information on a given IP."""
if input_file is None and not sys.stdin.isatty():
input_file = click.open_file("-")
if input_file is None and not ip_address:
click.echo(context.get_help())
context.exit(-1)
ip_addresses = []
if input_file is not None:
lines = [line.strip() for line in input_file]
ip_addresses.extend(
[line for line in lines if validate_ip(line, strict=False)])
if ip_address:
ip_addresses.append(ip_address)
if not ip_addresses:
output = [
context.command.get_usage(context),
("Error: at least one valid IP address must be passed either as an "
"argument (IP_ADDRESS) or through the -i/--input_file option."),
]
click.echo("\n\n".join(output))
context.exit(-1)
results = [
api_client.ip(ip_address=ip_address) for ip_address in ip_addresses
]
return results
def ip_address_parameter(_context, _parameter, value):
"""IPv4 address passed from the command line.
:param value: IPv4 address value
:type value: str
:raises click.BadParameter: when IP address value is invalid
"""
if value is None:
return value
try:
validate_ip(value)
except ValueError:
raise click.BadParameter(value)
return value
def interesting(self, ip_address):
"""Report an IP as "interesting".
:param ip_address: IP address to report as "interesting".
:type ip_address: str
"""
if self.offering == "community":
response = {
"message":
"Interesting report not supported with Community offering"
}
return response
else:
LOGGER.debug("Reporting interesting IP: %s...",
ip_address,
ip_address=ip_address)
validate_ip(ip_address)
endpoint = self.EP_INTERESTING.format(ip_address=ip_address)
response = self._request(endpoint, method="post")
return response
def transform(self, records):
"""Method that processes and yield event records to the Splunk events pipeline."""
ip_addresses = self.ip
ip_field = self.ip_field
api_key = ""
EVENTS_PER_CHUNK = 5000
THREADS = 3
USE_CACHE = False
logger = utility.setup_logger(
session_key=self._metadata.searchinfo.session_key,
log_context=self._metadata.searchinfo.command)
if ip_addresses and ip_field:
logger.error(
"Please use parameter ip to work gnquick as generating command or "
"use parameter ip_field to work gnquick as transforming command."
)
self.write_error(
"Please use parameter ip to work gnquick as generating command or "
"use parameter ip_field to work gnquick as transforming command"
)
exit(1)
try:
message = ''
api_key = utility.get_api_key(
self._metadata.searchinfo.session_key, logger=logger)
except APIKeyNotFoundError as e:
message = str(e)
except HTTPError as e:
message = str(e)
if message:
self.write_error(message)
logger.error(
"Error occured while retrieving API key, Error: {}".format(
message))
exit(1)
if ip_addresses and not ip_field:
# This peice of code will work as generating command and will not use the Splunk events.
# Splitting the ip_addresses by commas and stripping spaces from both the sides for each IP address
ip_addresses = [ip.strip() for ip in ip_addresses.split(',')]
logger.info("Started retrieving results")
try:
logger.debug(
"Initiating to fetch noise and RIOT status for IP address(es): {}"
.format(str(ip_addresses)))
api_client = GreyNoise(api_key=api_key,
timeout=120,
integration_name=INTEGRATION_NAME)
# CACHING START
cache_enabled, cache_client = utility.get_caching(
self._metadata.searchinfo.session_key, 'multi', logger)
if int(cache_enabled) == 1 and cache_client is not None:
cache_start = time.time()
ips_not_in_cache, ips_in_cache = utility.get_ips_not_in_cache(
cache_client, ip_addresses, logger)
try:
response = []
if len(ips_in_cache) >= 1:
response = cache_client.query_kv_store(
ips_in_cache)
if response is None:
logger.debug(
"KVStore is not ready. Skipping caching mechanism."
)
noise_status = api_client.quick(ip_addresses)
elif response == []:
noise_status = utility.fetch_response_from_api(
api_client.quick, cache_client, ip_addresses,
logger)
else:
noise_status = utility.fetch_response_from_api(
api_client.quick, cache_client,
ips_not_in_cache, logger)
noise_status.extend(response)
except Exception:
logger.debug(
"An exception occurred while fetching response from cache.\n{}"
.format(traceback.format_exc()))
logger.debug(
"Generating command with caching took {} seconds.".
format(time.time() - cache_start))
else:
# Opting timout 120 seconds for the requests
noise_status = api_client.quick(ip_addresses)
logger.info("Retrieved results successfully")
# CACHING END
# Process the API response and send the noise and RIOT status information of IP with extractions
# to the Splunk, Using this flag to handle the field extraction issue in custom commands
# Only the fields extracted from the first event of generated by custom command
# will be extracted from all events
first_record_flag = True
# Flag to indicate whether erroneous IPs are present
erroneous_ip_present = False
for ip in ip_addresses:
for sample in noise_status:
if ip == sample['ip']:
yield event_generator.make_valid_event(
'quick', sample, first_record_flag)
if first_record_flag:
first_record_flag = False
logger.debug(
"Fetched noise and RIOT status for ip={} from GreyNoise API"
.format(str(ip)))
break
else:
erroneous_ip_present = True
try:
validate_ip(ip, strict=True)
except ValueError as e:
error_msg = str(e).split(":")
logger.debug(
"Generating noise and RIOT status for ip={} manually"
.format(str(ip)))
event = {'ip': ip, 'error': error_msg[0]}
yield event_generator.make_invalid_event(
'quick', event, first_record_flag)
if first_record_flag:
first_record_flag = False
if erroneous_ip_present:
logger.warn(
"Value of one or more IP address(es) is either invalid or non-routable"
)
self.write_warning(
"Value of one or more IP address(es) passed to {command_name} "
"is either invalid or non-routable".format(
command_name=str(
self._metadata.searchinfo.command)))
except RateLimitError:
logger.error(
"Rate limit error occured while fetching the context information for ips={}"
.format(str(ip_addresses)))
self.write_error(
"The Rate Limit has been exceeded. Please contact the Administrator"
)
except RequestFailure as e:
response_code, response_message = e.args
if response_code == 401:
msg = "Unauthorized. Please check your API key."
else:
# Need to handle this, as splunklib is unable to handle the exception with
# (400, {'error': 'error_reason'}) format
msg = (
"The API call to the GreyNoise platform have been failed "
"with status_code: {} and error: {}").format(
response_code,
response_message['error'] if isinstance(
response_message, dict) else response_message)
logger.error("{}".format(str(msg)))
self.write_error(msg)
except ConnectionError:
logger.error(
"Error while connecting to the Server. Please check your connection and try again."
)
self.write_error(
"Error while connecting to the Server. Please check your connection and try again."
)
except RequestException:
logger.error(
"There was an ambiguous exception that occurred while handling your Request. Please try again."
)
self.write_error(
"There was an ambiguous exception that occurred while handling your Request. Please try again."
)
except Exception:
logger.error("Exception: {} ".format(
str(traceback.format_exc())))
self.write_error(
"Exception occured while fetching the noise and RIOT status of the IP address(es). "
"See greynoise_main.log for more details.")
elif ip_field:
# Enter the mechanism only when the Search is complete and all the events are available
if self.search_results_info and not self.metadata.preview:
try:
# Strip the spaces from the parameter value if given
ip_field = ip_field.strip()
# Validating the given parameter
try:
ip_field = validator.Fieldname(
option_name='ip_field').validate(ip_field)
except ValueError as e:
# Validator will throw ValueError with error message when the parameters are not proper
logger.error(str(e))
self.write_error(str(e))
exit(1)
# API key validation
if not self.api_validation_flag:
api_key_validation, message = utility.validate_api_key(
api_key, logger)
logger.debug(
"API validation status: {}, message: {}".format(
api_key_validation, str(message)))
self.api_validation_flag = True
if not api_key_validation:
logger.info(message)
self.write_error(message)
exit(1)
# This piece of code will work as transforming command and will use
# the Splunk ingested events and field which is specified in ip_field.
chunk_dict = event_generator.batch(records, ip_field,
EVENTS_PER_CHUNK,
logger)
# This means there are only 1000 or below IPs to call in the entire bunch of records
# Use one thread with single thread with caching mechanism enabled for the chunk
if len(chunk_dict) == 1:
logger.info(
"Less then 1000 distinct IPs are present, "
"optimizing the IP requests call to GreyNoise API..."
)
THREADS = 1
USE_CACHE = True
api_client = GreyNoise(api_key=api_key,
timeout=120,
use_cache=USE_CACHE,
integration_name=INTEGRATION_NAME)
# When no records found, batch will return {0:([],[])}
tot_time_start = time.time()
if len(list(chunk_dict.values())[0][0]) >= 1:
for event in event_generator.get_all_events(
self._metadata.searchinfo.session_key,
api_client,
'multi',
ip_field,
chunk_dict,
logger,
threads=THREADS):
yield event
else:
logger.info(
"No events found, please increase the search timespan to have more search results."
)
tot_time_end = time.time()
logger.debug(
"Total execution time => {}".format(tot_time_end -
tot_time_start))
except Exception:
logger.info(
"Exception occured while adding the noise and RIOT status to the events, Error: {}"
.format(traceback.format_exc()))
self.write_error(
"Exception occured while adding the noise and RIOT status of "
"the IP addresses to events. See greynoise_main.log for more details."
)
else:
logger.error(
"Please specify exactly one parameter from ip and ip_field with some value."
)
self.write_error(
"Please specify exactly one parameter from ip and ip_field with some value."
)
def event_processor(records_dict, result, method, ip_field, logger):
"""
Process on each chunk, format response retrieved from API and Send the results of transforming command to Splunk.
:param records_dict: Tuple having all the records of the chunk and all the IP addresses present in the ip_field
:param method: method used for the API invocation
:param ip_field: name of the field representing the IP address in Splunk events
:param logger: logger instance
:return: dict denoting the event to send to Splunk
"""
generate_missing_events, result = method_response_mapper(
method, result, logger)
# Loading the response to avoid loading it each time
# This will either have API response for the chunk or
# the exception message denoting exception occured while fetching the data
if result['response'] != []:
if type(result['response'][0]) == list:
api_results = []
for each in result['response'][0]:
api_results.append(each)
else:
api_results = result['response']
else:
api_results = result['response']
error_flag = True
# Before yielding events, make the ip lookup dict which will have the following format:
# {<ip-address>: <API response for that IP address>}
ip_lookup = {}
if result['message'] == 'ok':
error_flag = False
for event in api_results:
ip_lookup[event['ip']] = event
# This will be called per chunk to yield the events as per the objective of trasnforming command
for record in records_dict[0]:
if error_flag:
# Exception has occured while fetching the data
if ip_field in record and record[ip_field]:
event = {'ip': record[ip_field], 'error': api_results}
yield make_invalid_event(method, event, True, record)
else:
# Either the record is not having IP field or the value of the IP field is ''
# send the record as it is as it doesn't have any IP address, after appending all fields
yield make_invalid_event(method, {}, True, record)
else:
# Successful execution of the API call
if ip_field in record and record[ip_field]:
# Check if the IP field is not an iterable to avoid any error while referencing ip in ip_lookup
if isinstance(
record[ip_field],
six.string_types) and record[ip_field] in ip_lookup:
# Deleting the raw_data from the response when the request method is enrich
if method == 'enrich' and 'raw_data' in ip_lookup[
record[ip_field]]:
del ip_lookup[record[ip_field]]['raw_data']
yield make_valid_event(method, ip_lookup[record[ip_field]],
True, record)
else:
# Meaning ip is either invalid or not returned by the API,
# happens when quick method is used while retrieving data
if generate_missing_events:
try:
validate_ip(record[ip_field], strict=True)
except ValueError as e:
error_msg = str(e).split(":")
event = {
'ip': record[ip_field],
'error': error_msg[0]
}
yield make_invalid_event(method, event, True,
record)
else:
# Either the record is not having IP field or the value of the IP field is ''
# send the record as it is as it doesn't have any IP address, after appending all fields
yield make_invalid_event(method, {}, True, record)
def quick(self, ip_addresses):
"""Get activity associated with one or more IP addresses.
:param ip_addresses: One or more IP addresses to use in the look-up.
:type ip_addresses: str | list
:return: Bulk status information for IP addresses.
:rtype: dict
"""
LOGGER.debug("Getting noise status for %s...", ip_addresses)
if isinstance(ip_addresses, str):
ip_addresses = [ip_addresses]
ip_addresses = [
ip_address for ip_address in ip_addresses
if validate_ip(ip_address, strict=False)
]
if self.use_cache:
cache = self.IP_QUICK_CHECK_CACHE
# Keep the same ordering as in the input
ordered_results = OrderedDict((ip_address, cache.get(ip_address))
for ip_address in ip_addresses)
api_ip_addresses = [
ip_address for ip_address, result in ordered_results.items()
if result is None
]
if api_ip_addresses:
api_results = []
if len(api_ip_addresses) == 1:
endpoint = self.EP_NOISE_QUICK.format(
ip_address=api_ip_addresses[0])
api_results.append(self._request(endpoint))
else:
for chunk in more_itertools.chunked(
api_ip_addresses, self.IP_QUICK_CHECK_CHUNK_SIZE):
api_results.extend(
self._request(self.EP_NOISE_MULTI,
json={"ips": chunk}))
for api_result in api_results:
ip_address = api_result["ip"]
ordered_results[ip_address] = cache.setdefault(
ip_address, api_result)
results = list(ordered_results.values())
else:
results = []
if len(ip_addresses) == 1:
endpoint = self.EP_NOISE_QUICK.format(
ip_address=ip_addresses[0])
results.append(self._request(endpoint))
else:
for chunk in more_itertools.chunked(
ip_addresses, self.IP_QUICK_CHECK_CHUNK_SIZE):
results.extend(
self._request(self.EP_NOISE_MULTI, json={"ips":
chunk}))
for result in results:
code = result["code"]
result["code_message"] = self.CODE_MESSAGES.get(
code, self.UNKNOWN_CODE_MESSAGE.format(code))
return results
def test_invalid(self, ip):
"""Invalid ip address values."""
with pytest.raises(ValueError) as exception:
validate_ip(ip)
assert str(exception.value) == "Invalid IP address: {!r}".format(ip)
def quick(self, ip_addresses, include_invalid=False): # noqa: C901
"""Get activity associated with one or more IP addresses.
:param ip_addresses: One or more IP addresses to use in the look-up.
:type ip_addresses: str | list
:return: Bulk status information for IP addresses.
:rtype: dict
:param include_invalid: True or False
:type include_invalid: bool
"""
if self.offering == "community":
response = [
{"message": "Quick Lookup not supported with Community offering"}
]
return response
else:
if isinstance(ip_addresses, str):
ip_addresses = ip_addresses.split(",")
LOGGER.debug("Getting noise status...", ip_addresses=ip_addresses)
valid_ip_addresses = [
ip_address
for ip_address in ip_addresses
if validate_ip(ip_address, strict=False)
]
if self.use_cache:
cache = self.ip_quick_check_cache
# Keep the same ordering as in the input
ordered_results = OrderedDict(
(ip_address, cache.get(ip_address))
for ip_address in valid_ip_addresses
)
api_ip_addresses = [
ip_address
for ip_address, result in ordered_results.items()
if result is None
]
if api_ip_addresses:
api_results = []
chunks = more_itertools.chunked(
api_ip_addresses, self.IP_QUICK_CHECK_CHUNK_SIZE
)
for chunk in chunks:
api_result = self._request(
self.EP_NOISE_MULTI, json={"ips": chunk}
)
if isinstance(api_result, list):
api_results.extend(api_result)
else:
api_results.append(api_result)
for api_result in api_results:
ip_address = api_result["ip"]
ordered_results[ip_address] = cache.setdefault(
ip_address, api_result
)
results = list(ordered_results.values())
else:
results = []
chunks = more_itertools.chunked(
valid_ip_addresses, self.IP_QUICK_CHECK_CHUNK_SIZE
)
for chunk in chunks:
result = self._request(self.EP_NOISE_MULTI, json={"ips": chunk})
if isinstance(result, list):
results.extend(result)
else:
results.append(result)
[
results.append({"ip": ip, "noise": False, "code": "404"})
for ip in ip_addresses
if ip not in valid_ip_addresses and include_invalid
]
for result in results:
code = result["code"]
result["code_message"] = self.CODE_MESSAGES.get(
code, self.UNKNOWN_CODE_MESSAGE.format(code)
)
return results
def test_valid(self, ip):
"""Valid ip address values."""
validate_ip(ip)
