class APIClient():
PING_TIMEOUT_IN_SECONDS = 10
def __init__(self, connection, configuration):
self.endpoint_start = 'alertflex-ctrl/rest/stix-alerts'
headers = dict()
auth = configuration.get('auth')
headers['Authorization'] = b"Basic " + base64.b64encode(
(auth['username'] + ':' + auth['password']).encode('ascii'))
url_modifier_function = None
self.client = RestApiClient(connection.get('host'),
connection.get('port'),
headers,
url_modifier_function,
cert_verify=connection.get(
'selfSignedCert', False))
def ping_data_source(self):
endpoint = self.endpoint_start + '/status'
return self.client.call_api(endpoint,
'GET',
timeout=self.PING_TIMEOUT_IN_SECONDS)
def get_search_results(self, query_expression, offset=None, length=None):
endpoint = self.endpoint_start + '/search'
data = {'query': query_expression}
result = self.client.call_api(endpoint, 'GET', urldata=data)
return result
def delete_search(self, search_id):
# Optional since this may not be supported by the data source API
# Delete the search
return {"code": 200, "success": True}
class APIClient():
PING_ENDPOINT = 'sensor'
PROCESS_ENDPOINT = 'process'
def __init__(self, connection, configuration):
self.endpoint_start = 'api/v1/'
auth = configuration.get('auth')
headers = dict()
headers['X-Auth-Token'] = auth.get('token')
self.client = RestApiClient(connection.get('host'),
connection.get('port'),
headers,
cert_verify=connection.get(
'selfSignedCert', True),
sni=connection.get('sni', None))
self.timeout = connection['options'].get('timeout')
def ping_box(self):
endpoint = self.endpoint_start + self.PING_ENDPOINT
return self.client.call_api(endpoint, 'GET', timeout=self.timeout)
def run_search(self, query_expression, start=0, rows=10):
headers = dict()
endpoint = self.endpoint_start + self.PROCESS_ENDPOINT
data = [("q", query_expression), ("start", start), ("rows", rows)]
sort_by = 'start asc' # The purpose of this is to maintain order stability when doing paging
data.append(("sort", sort_by))
return self.client.call_api(endpoint,
'GET',
headers,
urldata=data,
timeout=self.timeout)
class APIClient:
"""API Client to handle all calls."""
PING_TIMEOUT_IN_SECONDS = 10
def __init__(self, connection, configuration):
"""Initialization.
:param connection: dict, connection dict
:param configuration: dict,config dict"""
headers = dict()
url_modifier_function = None
auth = configuration.get('auth')
self.endpoint = 'api/advancedqueries/run'
self.host = connection.get('host')
if auth:
if 'access_token' in auth:
headers['Authorization'] = "Bearer " + auth['access_token']
self.client = RestApiClient(
connection.get('host'),
connection.get('port', None),
connection.get('cert', None),
headers,
url_modifier_function=url_modifier_function,
cert_verify=connection.get('selfSignedCert', True),
mutual_auth=connection.get('use_securegateway', False),
sni=connection.get('sni', None))
def ping_box(self):
"""Ping the endpoint."""
endpoint = '/api'
return self.client.call_api(endpoint,
'GET',
timeout=self.PING_TIMEOUT_IN_SECONDS)
def run_search(self,
query_expression,
offset=DEFAULT_OFFSET,
length=DEFAULT_LIMIT):
"""get the response from MSatp endpoints
:param query_expression: str, search_id
:param offset: int,offset value
:param length: int,length value
:return: response, json object"""
serialize = '| serialize rn = row_number() | where rn >= {offset} | limit {length}'
headers = dict()
headers['Content-Type'] = 'application/json'
headers['Accept'] = 'application/json'
endpoint = self.endpoint
query_expression = query_expression + serialize.format(offset=offset,
length=length)
query_expression = json.dumps({
'Query': query_expression
}).encode("utf-8")
return self.client.call_api(endpoint,
'POST',
headers=headers,
data=query_expression)
class APIClient:
"""API Client to handle all calls."""
def __init__(self, connection, configuration):
"""Initialization.
:param connection: dict, connection dict
:param configuration: dict,config dict"""
headers = dict()
url_modifier_function = None
default_api_version = 'v1.0'
auth = configuration.get('auth')
self.endpoint = '{api_version}/security/alerts'.format(api_version=default_api_version)
self.host = connection.get('host')
self.timeout = connection['options'].get('timeout')
if auth:
if 'access_token' in auth:
headers['Authorization'] = "Bearer " + auth['access_token']
self.client = RestApiClient(connection.get('host'),
connection.get('port', None),
headers,
url_modifier_function=url_modifier_function,
cert_verify=connection.get('selfSignedCert', True),
sni=connection.get('sni', None)
)
def ping_box(self):
"""Ping the endpoint."""
params = dict()
params['$top'] = 1
return self.client.call_api(self.endpoint, 'GET', urldata=params, timeout=self.timeout)
def run_search(self, query_expression, length):
"""get the response from azure_sentinel endpoints
:param query_expression: str, search_id
:param length: int,length value
:return: response, json object"""
headers = dict()
headers['Accept'] = 'application/json'
params = dict()
params['$filter'] = query_expression
params['$top'] = length
return self.client.call_api(self.endpoint, 'GET', headers, urldata=params, timeout=self.timeout)
def next_page_run_search(self, next_page_url):
"""get the response from azure_sentinel endpoints
:param next_page_url: str, search_id
:return: response, json object"""
headers = dict()
headers['Accept'] = 'application/json'
url = next_page_url.split('?', maxsplit=1)[1]
endpoint = self.endpoint + '?' + url
return self.client.call_api(endpoint, 'GET', headers, timeout=self.timeout)
class APIClient():
PING_ENDPOINT = 'sensor'
PROCESS_ENDPOINT = 'process'
def __init__(self, connection, configuration):
self.endpoint_start_v1 = 'api/v1/' # Uses API v1 for `ping` and `processes search` endpoints.
self.endpoint_start_v4 = 'api/v4/' # Uses API v4 for `events search` endpoint.
auth = configuration.get('auth')
headers = dict()
headers['X-Auth-Token'] = auth.get('token')
self.client = RestApiClient(connection.get('host'),
connection.get('port'),
headers,
cert_verify=connection.get(
'selfSignedCert', True),
sni=connection.get('sni', None))
self.timeout = connection['options'].get('timeout')
def ping_box(self):
endpoint = self.endpoint_start_v1 + self.PING_ENDPOINT
return self.client.call_api(endpoint, 'GET', timeout=self.timeout)
def run_processes_search(self, query_expression, start=0, rows=10):
"""
https://developer.carbonblack.com/reference/enterprise-response/6.3/rest-api/#process-search
Processes search using `/api/v1/process`
"""
headers = dict()
process_endpoint = self.endpoint_start_v1 + self.PROCESS_ENDPOINT
data = [("q", query_expression), ("start", start), ("rows", rows),
("sort", 'start asc')]
return self.client.call_api(process_endpoint,
'GET',
headers,
urldata=data,
timeout=self.timeout)
def run_events_search(self, process_id, segment_id):
"""
https://developer.carbonblack.com/reference/enterprise-response/6.3/rest-api/#process-event-details
Event details search for process X at segment Y using `/api/v4/process/(process_id)/(segment_id)/event`
"""
headers = dict()
event_endpoint = self.endpoint_start_v4 + self.PROCESS_ENDPOINT + '/{}/{}/event'.format(
process_id, segment_id)
data = []
return self.client.call_api(event_endpoint,
'GET',
headers,
urldata=data,
timeout=self.timeout)
def parse_query(self, data):
proxy_host = self.options['proxy_host']
proxy_port = self.options['proxy_port']
connection, configuration = unwrap_connection_options(self.options)
client = RestApiClient(
proxy_host,
proxy_port,
url_modifier_function=lambda host_port, endpoint, headers:
f'https://{host_port}{endpoint}',
cert_verify=self.options.get('proxy_cert'))
response = client.call_api('/parse_query',
'POST',
data=json.dumps({
'module':
connection['type'],
'data_source': {},
'data':
data,
'options':
connection['options']
}),
timeout=self.options.get('timeout'))
return json.loads(response.bytes)
def translate_results(self, data_source, data):
# A proxy translation call passes the entire data source connection object in as the options
# Top-most connection host and port are for the proxy
proxy_host = self.options['proxy_host']
proxy_port = self.options['proxy_port']
connection, configuration = unwrap_connection_options(self.options)
client = RestApiClient(
proxy_host,
proxy_port,
url_modifier_function=lambda host_port, endpoint, headers:
f'http://{host_port}{endpoint}')
response = client.call_api('/translate_results',
'POST',
data=json.dumps({
'module':
connection['type'],
"data_source":
data_source,
"results":
data,
"options":
connection['options']
}),
timeout=self.options.get('timeout'))
return json.loads(response.bytes)
def get_Secret(self):
eventDetail = self.get_events()
secretIdList = []
secretCollection = []
for obj in eventDetail:
item = (obj['ItemId'])
secretIdList.append(item)
unique = set(secretIdList)
for id in unique:
secret_server_user_url = self.secret_detail + "/%s" % id
headers = {
'Authorization': self.accessToken,
'Content-Type': 'application/json'
}
payload = {}
response = RestApiClient.call_api(self,
secret_server_user_url,
'GET',
headers=headers,
data=payload,
urldata=None,
timeout=None)
secretCollection.append(response.response.text)
json_data = json.dumps(secretCollection)
collection = json.loads(json_data)
return collection
def get_events(self):
payload = "{\"name\": \"Secret Server Events Logs\", \"parameters\": [{\"name\": \"startDate\", \"value\": '%s'} , {\"name\":\"endDate\",\"value\": '%s'}]}" % (
self.startDate, self.endDate)
headers = {
'Authorization': self.accessToken,
'Content-Type': 'application/json'
}
endpoint = "SecretServer/api/v1/reports/execute"
response = RestApiClient.call_api(self,
endpoint,
'POST',
headers=headers,
data=payload,
urldata=None,
timeout=None)
return_obj = {}
if response.code != 200:
response_txt = response.response.text
ErrorResponder.fill_error(return_obj, message=response_txt)
raise Exception(return_obj)
collection = []
json_data = response.response.text
eventData = json.loads(json_data)
col = eventData['columns']
for obj in eventData['rows']:
obj = dict(zip(col, obj))
collection.append(obj)
return collection
class APIClient():
PING_ENDPOINT = 'sensor'
PING_TIMEOUT_IN_SECONDS = 10
@staticmethod
def _dialect_to_endpoint(dialect):
assert dialect in ["binary", "process"]
return dialect
def __init__(self, connection, configuration):
self.endpoint_start = 'api/v1/'
auth = configuration.get('auth')
headers = dict()
headers['X-Auth-Token'] = auth.get('token')
self.client = RestApiClient(connection.get('host'),
connection.get('port'),
headers,
cert_verify=connection.get(
'selfSignedCert', True),
sni=connection.get('sni', None))
self.search_timeout = connection['options'].get('timeout')
def ping_box(self):
endpoint = self.endpoint_start + self.PING_ENDPOINT
return self.client.call_api(endpoint,
'GET',
timeout=self.PING_TIMEOUT_IN_SECONDS)
def run_search(self, query_expression, dialect, start=0, rows=10):
headers = dict()
endpoint = self.endpoint_start + self._dialect_to_endpoint(dialect)
data = [("q", query_expression), ("start", start), ("rows", rows)]
# The purpose of this is to maintain order stability when doing paging
if dialect == 'binary':
sort_by = 'server_added_timestamp asc'
else: # process
sort_by = 'start asc'
data.append(("sort", sort_by))
return self.client.call_api(endpoint,
'GET',
headers,
urldata=data,
timeout=self.search_timeout)
class APIClient():
def __init__(self, connection, configuration):
# Uncomment when implementing data source API client.
auth = configuration.get('auth')
headers = dict()
if auth:
if 'principal' in auth and 'secret' in auth:
headers['Authorization'] = b"Basic " + base64.b64encode(
(auth['principal'] + ':' + auth['secret']).encode('ascii'))
self.client = RestApiClient(connection.get('host'),
port=None,
headers=headers,
url_modifier_function=None,
cert_verify=True,
sni=None,
auth=None)
def ping_data_source(self):
# Pings the data source
endpoint = ENDPOINT_ALL + "?format=json&sinceSeconds=3600"
pingresult = self.client.call_api(endpoint=endpoint, method='GET')
return pingresult
def create_search(self, query_expression):
# Queries the data source
return {"code": 200, "query_id": query_expression}
def get_search_status(self, search_id):
# Check the current status of the search
return {"code": 200, "status": "COMPLETED"}
def get_search_results(self, search_id):
# Return the search results. Results must be in JSON format before being translated into STIX
#resultdata = self.client.call_api(endpoint=ENDPOINT_ALL+search_id, method='GET')#working
endpoint = ENDPOINT_ALL + "?format=json"
resultdata = self.client.call_api(endpoint=endpoint,
method='GET',
urldata=search_id)
# Check the current status of the search
return resultdata
def delete_search(self, search_id):
# Optional since this may not be supported by the data source API
# Delete the search
return {"code": 200, "success": True}
def ping_data_source(self):
response = RestApiClient.call_api(self,
self.auth_token_url,
'GET',
headers=self.headers,
data=self.payload,
urldata=None,
timeout=None)
return response.code
class Connector(BaseConnector):
def __init__(self, connection, configuration):
self.request_http_path = "http://{}:{}".format(connection['options']['proxy_host'], connection['options']['proxy_port'])
self.timeout = connection['options']['timeout']
self.connection, self.configuration = self._unwrap_connection_options(copy.deepcopy(connection), copy.deepcopy(configuration))
self.client = RestApiClient(connection['options']['proxy_host'], connection['options']['proxy_port'], url_modifier_function=lambda host_port,endpoint,headers: f'http://{host_port}{endpoint}')
def ping_connection(self):
data = json.dumps({"connection": self.connection, "configuration": self.configuration})
response = self.client.call_api('/ping', 'POST', data=data, timeout=self.timeout)
return json.loads(response.bytes)
def create_query_connection(self, query):
data = json.dumps({"connection": self.connection, "configuration": self.configuration, "query": query})
response = self.client.call_api('/create_query_connection', 'POST', data=data, timeout=self.timeout)
return json.loads(response.bytes)
def create_results_connection(self, search_id, offset, length):
data = json.dumps({"connection": self.connection, "configuration": self.configuration, "search_id": search_id, "offset": offset, "length": length})
response = self.client.call_api('/create_results_connection', 'POST', data=data, timeout=self.timeout)
return json.loads(response.bytes)
def create_status_connection(self, search_id):
data = json.dumps({"connection": self.connection, "configuration": self.configuration, "search_id": search_id})
response = self.client.call_api('/create_status_connection', 'POST', data=data, timeout=self.timeout)
return json.loads(response.bytes)
def delete_query_connection(self, search_id):
data = json.dumps({"connection": self.connection, "configuration": self.configuration, "search_id": search_id})
response = self.client.call_api('/delete_query_connection', 'POST', data=data, timeout=self.timeout)
return json.loads(response.bytes)
def is_async(self):
data = json.dumps({"connection": self.connection, "configuration": self.configuration})
response = self.client.call_api('/is_async', 'POST', data=data, timeout=self.timeout)
return json.loads(response.bytes)
def _unwrap_connection_options(self, connection, configuration):
if 'options' in connection and 'destination' in connection['options']:
destination_params = connection['options']['destination']
if type(destination_params) == str:
if len(destination_params):
destination_params = json.loads(destination_params)
else:
destination_params = {}
if destination_params:
return destination_params['connection'], destination_params['configuration']
return connection, configuration
def get_token(self):
response = RestApiClient.call_api(self,
self.auth_token_url,
'GET',
headers=self.headers,
data=self.payload,
urldata=None,
timeout=None)
return_obj = {}
response_code = response.code
response_txt = response.response.text
if (response_code == 200):
json_obj = json.loads(response_txt)
token = json_obj.get('access_token')
self.accessToken = 'Bearer' + " " + token
return self.accessToken
else:
ErrorResponder.fill_error(return_obj, message=response_txt)
raise Exception(return_obj)
class Connector(BaseSyncConnector):
def __init__(self, connection, configuration):
self.connection = connection
self.configuration = configuration
self.timeout = connection['options'].get('timeout')
self.bundle_url = self.connection.get('url')
auth = None
conf_auth = configuration.get('auth', {})
if 'username' in conf_auth and 'password' in conf_auth:
auth = (conf_auth['username'], conf_auth['password'])
self.client = RestApiClient(None,
auth=auth,
url_modifier_function=lambda host_port,
endpoint, headers: f'{endpoint}')
# We re-implement this method so we can fetch all the "bindings", as their method only
# returns the first for some reason
def match(self, pattern, observed_data_sdos, verbose=False):
compiled_pattern = Pattern(pattern)
matcher = MatchListener(observed_data_sdos, verbose)
compiled_pattern.walk(matcher)
found_bindings = matcher.matched()
if found_bindings:
matching_sdos = []
for binding in found_bindings:
matches = [
match for match in matcher.get_sdos_from_binding(binding)
if match not in matching_sdos
]
matching_sdos.extend(matches)
else:
matching_sdos = []
return matching_sdos
def ping_connection(self):
return_obj = dict()
response = self.client.call_api(self.bundle_url,
'head',
timeout=self.timeout)
response_txt = response.raise_for_status()
if response.code == 200:
return_obj['success'] = True
elif response.code == 301:
self.bundle_url = response.headers.get('Location')
return self.ping_connection()
else:
ErrorResponder.fill_error(return_obj, response_txt, ['message'])
return return_obj
def create_results_connection(self, search_id, offset, length):
observations = []
return_obj = dict()
response = None
if self.connection['options'].get('error_type') == ERROR_TYPE_TIMEOUT:
# httpstat.us/200?sleep=60000 for slow connection that is valid
response = self.client.call_api(
'https://httpstat.us/200?sleep=60000',
'get',
timeout=self.timeout)
elif self.connection['options'].get(
'error_type') == ERROR_TYPE_BAD_CONNECTION:
# www.google.com:81 for a bad connection that will timeout
response = self.client.call_api('https://www.google.com:81',
'get',
timeout=self.timeout)
else:
response = self.client.call_api(self.bundle_url,
'get',
timeout=self.timeout)
if response.code != 200:
response_txt = response.raise_for_status()
if ErrorResponder.is_plain_string(response_txt):
ErrorResponder.fill_error(return_obj, message=response_txt)
elif ErrorResponder.is_json_string(response_txt):
response_json = json.loads(response_txt)
ErrorResponder.fill_error(return_obj, response_json,
['reason'])
else:
raise UnexpectedResponseException
else:
try:
response_txt = response.read().decode('utf-8')
bundle = json.loads(response_txt)
if "stix_validator" in self.connection[
'options'] and self.connection['options'].get(
"stix_validator") is True:
results = validate_instance(bundle)
if results.is_valid is not True:
ErrorResponder.fill_error(
return_obj,
message='Invalid Objects in STIX Bundle.')
return return_obj
for obj in bundle["objects"]:
if obj["type"] == "observed-data":
observations.append(obj)
# Pattern match
try:
results = self.match(search_id, observations, False)
if len(results) != 0:
return_obj['success'] = True
return_obj['data'] = results[int(offset):int(offset +
length)]
else:
return_obj['success'] = True
return_obj['data'] = []
except Exception as ex:
ErrorResponder.fill_error(
return_obj,
message='Object matching error: ' + str(ex))
except Exception as ex:
ErrorResponder.fill_error(
return_obj,
message='Invalid STIX bundle. Malformed JSON: ' + str(ex))
return return_obj
def create_query_connection(self, query):
return {"success": True, "search_id": query}
def create_status_connection(self, search_id):
return {"success": True, "status": "COMPLETED", "progress": 100}
def delete_query_connection(self, search_id):
return_obj = dict()
return_obj['success'] = True
return return_obj
class APIClient():
def __init__(self, connection, configuration):
auth = configuration.get('auth')
headers = dict()
headers['X-Auth-Token'] = auth.get('token')
headers['Accept'] = 'application/json'
headers['Content-Type'] = 'application/json'
self.org_key = auth.get('org_key')
self.client = RestApiClient(connection.get('host'),
connection.get('port'),
headers,
cert_verify=connection.get(
'selfSignedCert', True),
sni=connection.get('sni', None))
self.timeout = connection['options'].get('timeout')
self.result_limit = connection['options'].get('result_limit')
def ping_data_source(self):
"""Verifies the data source API is working by sending a GET request to
https://<server_ip>/api/investigate/v1/orgs/{org_key}/processes/limits
Status codes:
200: successfully fetched the upper and lower time limits (i.e API works)
400: malformed JSON body or invalid value
403: forbidden
500: internal server error
"""
endpoint = f'api/investigate/v1/orgs/{self.org_key}/processes/limits'
return self.client.call_api(endpoint, 'GET', timeout=self.timeout)
def create_search(self, query_expression):
"""Queries the data source by sending a POST request to
https://<server_ip>/api/investigate/v2/orgs/{org_key}/processes/search_jobs
200: successfully submitted search for processes
400: malformed JSON body or invalid value
403: forbidden
500: internal server error
"""
endpoint = f'api/investigate/v2/orgs/{self.org_key}/processes/search_jobs'
data = {
'query': query_expression,
'fields': DEFAULT_FIELDS,
'start': 0,
'rows': self.result_limit,
'sort': [{
'field': 'device_timestamp',
'order': 'asc'
}]
}
return self.client.call_api(endpoint,
'POST',
data=json.dumps(data),
timeout=self.timeout)
def get_search_status(self, job_id):
""" Check the status of the search by sending a GET request to
https://<server_ip>/api/investigate/v1/orgs/{org_key}/processes/search_jobs/{job_id}
200: successfully retrieved status of process search
400: malformed JSON body or invalid value
403: forbidden
500: internal server error
"""
endpoint = f'api/investigate/v1/orgs/{self.org_key}/processes/search_jobs/{job_id}'
return self.client.call_api(endpoint, 'GET', timeout=self.timeout)
def get_search_results(self, job_id, start=0, rows=100):
"""Return the JSON-formatted search results by sending a GET request to
https://<server_ip>/api/investigate/v2/orgs/{org_key}/processes/search_jobs/{job_id}/results
200: successfully fetched processes
400: malformed JSON body or invalid value
403: forbidden
500: internal server error
"""
urldata = [("start", start), ("rows", rows)]
endpoint = f'api/investigate/v2/orgs/{self.org_key}/processes/search_jobs/{job_id}/results'
return self.client.call_api(endpoint,
'GET',
urldata=urldata,
timeout=self.timeout)
def delete_search(self, job_id):
"""Delete the search by sending a DELETE request to
https://<server_ip>/api/investigate/v1/orgs/{orgkey}/processes/search_jobs/{job_id}
204: success deleted a process search
400: malformed JSON body or invalid value
403: forbidden
500: internal server error
"""
endpoint = f'api/investigate/v1/orgs/{self.org_key}/processes/search_jobs/{job_id}'
return self.client.call_api(endpoint, 'DELETE', timeout=self.timeout)
class APIClient():
# API METHODS
# These methods are used to retrive a) authorization token using
# a rest api (sending credentials as params)
# b) reports (sending token and report query) using a different rest api
# This class will encode any data or query parameters which will then be
# sent to the call_api() method of the RestApiClient
def __init__(self, connection, configuration):
#
self.endpoint_start = 'restAPI/'
self.connection = connection
self.configuration = configuration
self.headers = dict()
self.search_id = None
self.query = None
self.authorization = None
self.credential = None
#
# Check if connection object contains the following
username = configuration.get('auth').get("username", None)
password = configuration.get('auth').get("password", None)
grant_type = connection.get('options', {}).get("grant_type",
'password')
client_id = connection.get('options', {}).get("client_id", None)
client_secret = connection.get('options',
{}).get("client_secret", None)
# It is decided the authorization will not be sent by UDS
#
if (username is None or password is None or grant_type is None
or client_id is None or client_secret is None):
self.credential = None
raise IOError(
3001,
"Guardium Credential not provided in the connection / configuration object"
)
else:
self.credential = {
"username": username,
"password": password,
"grant_type": grant_type,
"client_id": client_id,
"client_secret": client_secret
}
#
host_port = connection.get('host') + ':' + \
str(connection.get('port', ''))
url_modifier_function = None
proxy = connection.get('proxy')
if proxy is not None:
proxy_url = proxy.get('url')
proxy_auth = proxy.get('auth')
if (proxy_url is not None and proxy_auth is not None):
self.headers['proxy'] = proxy_url
self.headers['proxy-authorization'] = 'Basic ' + proxy_auth
if proxy.get('x_forward_proxy', None) is not None:
self.headers['x-forward-url'] = 'https://' + \
host_port + '/' # + endpoint, is set by 'add_endpoint_to_url_header'
host_port = proxy.get('x_forward_proxy')
if proxy.get('x_forward_proxy_auth', None) is not None:
self.headers['x-forward-auth'] = proxy.get(
'x_forward_proxy_auth')
self.headers['user-agent'] = 'UDS'
url_modifier_function = self.add_endpoint_to_url_header
self.client = RestApiClient(
host_port,
None,
connection.get('cert', None),
self.headers,
url_modifier_function,
cert_verify=connection.get('selfSignedCert', True),
mutual_auth=connection.get('use_securegateway', False),
sni=connection.get('sni', None))
# self.client = RestApiClient(host_port,None,connection.get('cert', None),self.headers,
# url_modifier_function,
# connection.get('cert_verify', 'True'))
#
def add_endpoint_to_url_header(self, url, endpoint, headers):
# this function is called from 'call_api' with proxy forwarding,
# it concatenates the endpoint to the header containing the url.
headers['x-forward-url'] += endpoint
# url is returned since it points to the proxy for initial call
return url
def add_header_elements(self, key, value):
self.headers[key] = value
return
def ping_box(self):
# Subroto -- Guardium does not have ping facility
# We test if we can get the access token if we can then success = true
#
respObj = Response()
if (self.fetch_accessToken()):
respObj.code = "200"
respObj.error_type = ""
respObj.status_code = 200
content = '{"status":"OK", "data": {"message": "Service is up."}}'
respObj._content = bytes(content, 'utf-8')
else:
respObj.code = "503"
respObj.error_type = "Service Unavailable"
respObj.status_code = 503
content = '{"status":"Failed", "data": {"message": "Service is down."}}'
respObj._content = bytes(content, 'utf-8')
# return
return ResponseWrapper(respObj)
#
def get_databases(self):
# Sends a GET request
endpoint = self.endpoint_start + 'databases'
return self.client.call_api(endpoint, 'GET')
def get_database(self, database_name):
# Sends a GET request
endpoint = self.endpoint_start + 'databases' + '/' + database_name
return self.client.call_api(endpoint, 'GET')
def isTimestampValid(self, tstamp):
if tstamp is not None:
if (tstamp > (datetime.datetime.now()).timestamp()):
return True
return False
#
def get_credential(self):
# Subroto -- Assumption: credential object will contain the full
# guardium credential for the call in json format.
if self.credential is None:
raise IOError(
3001,
"Guardium Credential object is found to be None. Error Raised."
)
#
else:
data = urllib.parse.urlencode(self.credential)
#
return data
#
def fetch_accessToken(self):
# process new authorization token
# Get access token if not present
# credential is a string contain a json
#
successVal = False
#
data = self.get_credential()
#print(data)
endpoint = "oauth/token"
tNow = datetime.datetime.now()
response = self.client.call_api(endpoint,
"POST",
params=data,
data=None)
jResp = json.loads(str(response.read(), 'utf-8'))
#
#print(jResp)
if (response.code != 200):
#print(response.code)
errMsg = str(jResp) + " -- " + "Authorization Token not received."
raise ValueError(3002, errMsg)
else:
successVal = True
tExp = (tNow + datetime.timedelta(seconds=jResp.get("expires_in"))
).timestamp()
self.authorization = json.loads('{"access_token":"' +
jResp.get("access_token") +
'", "expiresTimestamp":' +
str(tExp) + '}')
#
#print(self.authorization)
return successVal
#
def get_accessToken(self):
successVal = False
if (self.authorization is not None):
# Test for Authorization validity
if self.isTimestampValid(
(self.authorization).get("expiresTimestamp")):
successVal = True
self.setAuthorizationHeader()
return successVal
#
# We could not find a valid token from the and now we request one
#
if (self.fetch_accessToken()):
successVal = True
self.setAuthorizationHeader()
#
return successVal
#
def setAuthorizationHeader(self):
auth = "Bearer " + \
str((self.authorization).get("access_token"))
#
# Test -- uncomment to test token failure
#auth = "Bearer 575d161c-9a91-4c05-bf0a-1c2d415a8c40"
self.add_header_elements("authorization", auth)
return
#
# NOTE: connector architecture forces sync connector behave as async connector
# therefore, the state of the connector has to preserved in the search_id
# which is not generated by Guardium. We generate and store necessary status_code
# such as self.credential, self.query (may be self.authorization)
# IF search_id originally generated is changed then the state is lost
def set_searchId(self, search_id):
self.search_id = search_id
return
#
def build_searchId(self):
# It should be called only ONCE when transmit query is called
# Structure of the search id is
# '{"query": ' + json.dumps(self.query) + ', "credential" : ' + json.dumps(self.credential) + '}'
s_id = None
#
if (self.query is None or self.authorization is None
or self.credential is None):
raise IOError(
3001,
"Could not generate search id because 'query' or 'authorization token' or 'credential info' is not available."
)
#
else:
id_str = '{"query": ' + json.dumps(
self.query) + ', "credential" : ' + json.dumps(
self.credential) + '}'
#print(id_str)
id_byt = id_str.encode('utf-8')
s_id = base64.b64encode(id_byt).decode()
self.set_searchId(s_id)
#
#print(s_id)
return s_id
#
def decode_searchId(self):
# These value (self.credential, self.query) must be present. self.authorization may not.
try:
id_dec64 = base64.b64decode(self.search_id)
jObj = json.loads(id_dec64.decode('utf-8'))
except:
raise IOError(
3001, "Could not decode search id content - " + self.search_id)
#
self.query = jObj.get("query", None)
self.credential = jObj.get("credential", None)
self.authorization = jObj.get("authorization", None)
return
#
def get_searches(self):
# CAN NOT be implemented for Guardium
#
endpoint = self.endpoint_start + "searches"
return self.client.call_api(endpoint, 'GET')
#
def create_search(self, query_expression):
# validate credential and create search_id. No query submission -- Sync call
#
respObj = Response()
respObj.code = "401"
respObj.error_type = ""
respObj.status_code = 401
if (self.get_accessToken()):
self.query = query_expression
response = self.build_searchId()
if (response != None):
respObj.code = "200"
respObj.error_type = ""
respObj.status_code = 200
content = '{"search_id": "' + \
str(response) + \
'", "data": {"message": "Search id generated."}}'
respObj._content = bytes(content, 'utf-8')
else:
respObj.code = "404"
respObj.error_type = "Not found"
respObj.status_code = 404
respObj.message = "Could not generate search id."
else:
respObj.error_type = "Unauthorized: Access token could not be generated."
respObj.message = "Unauthorized: Access token could not be generated."
#
return ResponseWrapper(respObj)
#
def get_status(self, search_id):
# Subroto we do not need to send anything to Guardium
# We create response object and send "COMPLETED"
# Note: we may have an issue with this simplistic approach
respObj = Response()
if (self.fetch_accessToken()):
respObj.code = "200"
respObj.error_type = ""
respObj.status_code = 200
content = '{"search_id": "' + search_id + \
'", "progress":"Completed", "status":"COMPLETED", "data": {"message":"Completed for the search id provided."}}'
respObj._content = bytes(content, 'utf-8')
else:
respObj.code = "503"
respObj.error_type = "Service Unavailable"
respObj.status_code = 503
content = '{"status":"Failed", "data": {"message": "Could obtain status: Authentication issue / service unavailable."}}'
respObj._content = bytes(content, 'utf-8')
#
return ResponseWrapper(respObj)
#
def set_IndexAndFsize(self, indexFrom=None, fetchSize=None):
data = json.loads(self.query)
try:
indx = int(indexFrom)
fsize = int(fetchSize)
except ValueError:
print("Offset (indexFrom) or length (fetchSize) is not an integer")
#
# replace the data string
data["indexFrom"] = str(indx)
data["fetchSize"] = str(fsize)
return json.dumps(data)
#
def get_search_results(self,
search_id,
response_type,
indexFrom=None,
fetchSize=None):
# Sends a GET request from guardium
# This function calls Guardium to get data
self.set_searchId(search_id)
self.decode_searchId()
# replacement indexFrom and fetchSize
data = self.set_IndexAndFsize(indexFrom, fetchSize)
#
if (self.get_accessToken()):
endpoint = self.endpoint_start + "online_report"
#
response = self.client.call_api(endpoint,
'POST',
params=None,
data=data)
status_code = response.response.status_code
#
# Though the connector gets the authorization token just before fetching the actual result
# there is a possibility that the token returned is only valid for a second and response_code = 401
# is returned. Catch that situation (though remote) and process again.
if status_code != 200:
error_msg = json.loads(str(response.read(), 'utf-8'))
error_code = error_msg.get('error', None)
if status_code == 401 and error_code == "invalid_token":
self.authorization = None
if (self.get_accessToken()):
response = self.client.call_api(endpoint,
'POST',
params=None,
data=data)
status_code = response.response.status_code
else:
raise ValueError(3002,
"Authorization Token not received ")
#
# Now START and STOP are optional -- A situation can occur that data set can be empty -- handle this situation here
if status_code == 200:
#
# Determine if the response is empty if empty Guardium sends {"ID": 0,
# "Message": "ID=0 The Query did not retrieve any records"}
# Raise an error --> 1010: ErrorCode.TRANSMISSION_RESPONSE_EMPTY_RESULT
# response_content = self.raiseErrorIfEmptyResult(response)
return response
else:
raise ValueError(1020, "Error -- Status Code is NOT 200!")
else:
raise ValueError(3002, "Authorization Token not received ")
# End of this function
def raiseErrorIfEmptyResult(self, response):
# Determine if the response is empty if empty Guardium sends {"ID": 0,
# "Message": "ID=0 The Query did not retrieve any records"} <-- check that an raise and Error
# 1010: ErrorCode.TRANSMISSION_RESPONSE_EMPTY_RESULT
r_content_str = (response.read()).decode('utf8').replace("'", '"')
response_content = json.loads(r_content_str)
#print(r_content_str)
if "ID" in response_content:
#print(response_content)
errMsg = response_content.get(
"Message",
"Default Message - NO Records Fetched using this Query.")
raise ValueError(1010, errMsg)
else:
return response_content
#
#
def update_search(self, search_id, save_results=None, status=None):
# Subroto -- not used in Guardium context
# posts search result to site
endpoint = self.endpoint_start + "searches/" + search_id
data = {}
if save_results:
data['save_results'] = save_results
if status:
data['status'] = status
data = urllib.parse.urlencode(data)
data = data.encode('utf-8')
return self.client.call_api(endpoint, 'POST', params=None, data=data)
#
def delete_search(self, search_id):
# Subroto -- not used.
# deletes search created earlier.
#endpoint = self.endpoint_start + "searches" + '/' + search_id
#return self.client.call_api(endpoint, 'DELETE')
return {"success": True, "search_id": search_id}
class APIClient():
PING_ENDPOINT = '_cluster/health?pretty'
def __init__(self, connection, configuration):
self.logger = logger.set_logger(__name__)
headers = dict()
url_modifier_function = None
auth = configuration.get('auth')
self.indices = connection.get('indices', None)
if self.indices and type(self.indices) == str:
self.indices = self.indices.split(",")
if isinstance(self.indices, list): # Get list of all indices
self.indices = [i.strip(' ') for i in self.indices]
self.indices = ",".join(self.indices)
if self.indices:
self.endpoint = self.indices + '/' + '_search'
else:
self.endpoint = '_search'
if auth:
if 'username' in auth and 'password' in auth:
headers['Authorization'] = b"Basic " + base64.b64encode(
(auth['username'] + ':' +
auth['password']).encode('ascii'))
elif 'api_key' in auth and 'id' in auth:
headers['Authorization'] = b"ApiKey " + base64.b64encode(
(auth['id'] + ':' + auth['api_key']).encode('ascii'))
elif 'access_token' in auth:
headers['Authorization'] = "Bearer " + auth['access_token']
self.client = RestApiClient(
connection.get('host'),
connection.get('port'),
headers,
url_modifier_function=url_modifier_function,
cert_verify=connection.get('selfSignedCert', True),
sni=connection.get('sni', None))
self.timeout = connection['options'].get('timeout')
def ping_box(self):
return self.client.call_api(self.PING_ENDPOINT,
'GET',
timeout=self.timeout)
def run_search(self, query_expression, offset=None, length=DEFAULT_LIMIT):
headers = dict()
headers['Content-Type'] = 'application/json'
endpoint = self.endpoint
uri_search = False # For testing and debugging two ways of _search API methods
# URI Search
if uri_search:
if query_expression is not None:
# update/add size value
if length is not None:
if re.search(r"&size=\d+", query_expression):
query_expression = re.sub(r"(?<=&size=)\d+",
str(length),
query_expression)
else:
query_expression = '{}&size={}'.format(
query_expression, length)
# add offset to query expression
if offset is not None:
query_expression = '{}&from={}'.format(
query_expression, offset)
# addition of QueryString to API END point
endpoint = endpoint + '?q=' + query_expression
return self.client.call_api(endpoint,
'GET',
headers,
timeout=self.timeout)
# Request body search
else:
# add size value
if length is not None:
endpoint = "{}?size={}".format(endpoint, length)
# add offset value
if offset is not None:
endpoint = "{}&from={}".format(endpoint, offset)
data = {
"_source": {
"includes": [
"@timestamp", "source.*", "destination.*", "event.*",
"client.*", "server.*", "host.*", "network.*",
"process.*", "user.*", "file.*", "url.*", "registry.*",
"dns.*"
]
},
"query": {
"query_string": {
"query": query_expression
}
}
}
self.logger.debug("URL endpoint: " + endpoint)
self.logger.debug("URL data: " + json.dumps(data))
return self.client.call_api(endpoint,
'GET',
headers,
data=json.dumps(data),
timeout=self.timeout)
class APIClient:
QUERY_ENDPOINT = "advancedsearch/api/search/"
PING_ENDPOINT = "status?includechildren=false&fast=false"
def __init__(self, connection, configuration):
self.logger = logger.set_logger(__name__)
headers = {}
url_modifier_function = None
self.auth = configuration.get('auth')
self.client = RestApiClient(
connection.get('host'),
port=None,
headers=headers,
cert_verify=False,
url_modifier_function=url_modifier_function)
def ping_box(self):
"""
Ping the Data Source
:return: Response object
"""
encoded_query = self._encode_query("")
headers = self.get_header(self.PING_ENDPOINT, encoded_query)
return self.client.call_api(self.PING_ENDPOINT,
'GET',
headers=headers,
data=None)
def get_search_results(self, query):
"""
Get results from Data Source
:param query: Data Source Query
:return: Response Object
"""
self.logger.debug("query: %s", query)
encoded_query = self._encode_query(query)
headers = self.get_header(self.QUERY_ENDPOINT, encoded_query)
return self.client.call_api(self.QUERY_ENDPOINT + encoded_query,
'GET',
headers=headers,
data=None)
def get_header(self, endpoint, query):
query_url = "/" + endpoint + query
time = datetime.datetime.utcnow()
sig = hmac.new(
self.auth["private_token"].encode("ASCII"),
msg=(query_url + "\n" + self.auth["public_token"] + "\n" +
time.isoformat(timespec="auto")).encode("ASCII"),
digestmod=hashlib.sha1).hexdigest()
header = {
"DTAPI-Token": self.auth["public_token"],
"DTAPI-Date": time.isoformat(timespec="auto"),
"DTAPI-Signature": sig
}
return header
@staticmethod
def _encode_query(_query):
"""
Encode Query:
- Encode query to base64 and convert to string.
"""
_query_encode_bytes = base64.b64encode(bytes(_query, 'utf-8'))
_query_encoded = str(_query_encode_bytes, 'utf-8')
return _query_encoded
class APIClient():
QUERY_ENDPOINT = 'web/api/v2.1/dv/init-query'
RESULT_ENDPOINT = 'web/api/v2.1/dv/events'
QUERY_STATUS = 'web/api/v2.1/dv/query-status'
PING_STATUS = 'web/api/v2.1/system/status'
def __init__(self, connection, configuration):
headers = dict()
self.auth = configuration.get('auth')
self.api_key = "ApiToken " + self.auth.get('apitoken')
headers['Authorization'] = self.api_key
headers['Content-type'] = 'application/json'
self.timeout = connection['options']['timeout']
self.client = RestApiClient(connection.get('host'),
connection.get('port', None),
headers,
url_modifier_function=None)
def ping_datasource(self):
"""
ping or check the system status
"""
endpoint = self.PING_STATUS
return self.client.call_api(endpoint,
'GET',
headers=self.client.headers,
timeout=self.timeout)
def create_search(self, query_expression):
"""
init query
:param data source query
:return:queryId
"""
endpoint = self.QUERY_ENDPOINT
data = query_expression
data = data.encode('utf-8')
return self.client.call_api(endpoint,
'POST',
headers=self.client.headers,
data=data,
timeout=self.timeout)
def get_search_status(self, search_id):
"""
get query status
:param queryId:
:return:
"""
endpoint = self.QUERY_STATUS + "?queryId=" + search_id
params = {}
params['output'] = 'json'
return self.client.call_api(endpoint,
'GET',
headers=self.client.headers,
urldata=params,
timeout=self.timeout)
def get_search_results(self, search_id, offset, length, nextcursor=None):
"""
Get results from Data Source
:param query: Data Source QueryId,nextcursor,limit
:return: Response Object
"""
endpoint = self.RESULT_ENDPOINT + "?queryId=" + search_id
#Max limit of getting result in a api call is 1000
limit = 1000
if nextcursor is not None:
endpoint = endpoint + "&cursor=" + nextcursor
endpoint = endpoint + "&limit=" + str(limit)
params = {}
params['output'] = 'json'
return self.client.call_api(endpoint,
'GET',
headers=self.client.headers,
urldata=params,
timeout=self.timeout)
def delete_search(self, search_id):
"""
delete api is not supported
:param queryId:
:return:dict
"""
return {"code": 200, "success": True}
class APIClient():
# API METHODS
# These methods are used to call Splunk's API methods through http requests.
# Each method makes use of the http methods below to perform the requests.
# This class will encode any data or query parameters which will then be
# sent to the call_api() method of its inherited class.
PING_TIMEOUT_IN_SECONDS = 10
def __init__(self, connection, configuration):
# This version of the Splunk APIClient is designed to function with
# Splunk Enterprise version >= 6.5.0 and <= 7.1.2
# http://docs.splunk.com/Documentation/Splunk/7.1.2/RESTREF/RESTprolog
self.output_mode = 'json'
self.endpoint_start = 'services/'
self.authenticated = False
headers = dict()
self.client = RestApiClient(connection.get('host'),
connection.get('port'),
connection.get('cert', None),
headers,
cert_verify=connection.get('selfSignedCert', True),
mutual_auth=connection.get('use_securegateway', False),
sni=connection.get('sni', None)
)
self.auth = configuration.get('auth')
self.headers = headers
def authenticate(self):
if not self.authenticated:
self.set_splunk_auth_token(self.auth, self.headers)
self.authenticated = True
def set_splunk_auth_token(self, auth, headers):
data = {'username': auth['username'], 'password': auth['password'], 'output_mode': 'json'}
endpoint = self.endpoint_start + 'auth/login'
try:
response_json = json.load(self.client.call_api(endpoint, 'POST', headers, data=data))
headers['Authorization'] = "Splunk " + response_json['sessionKey']
except KeyError as e:
raise Exception('Authentication error occured while getting auth token: ' + str(e))
def ping_box(self):
self.authenticate()
endpoint = self.endpoint_start + 'server/status'
data = {'output_mode': self.output_mode}
return self.client.call_api(endpoint, 'GET', data=data, timeout=self.PING_TIMEOUT_IN_SECONDS)
def create_search(self, query_expression):
# sends a POST request to
# https://<server_ip>:<port>/services/search/jobs
self.authenticate()
endpoint = self.endpoint_start + "search/jobs"
data = {'search': query_expression, 'output_mode': self.output_mode}
return self.client.call_api(endpoint, 'POST', data=data)
def get_search(self, search_id):
# sends a GET request to
# https://<server_ip>:<port>/services/search/jobs/<search_id>
# returns information about the search job and its properties.
self.authenticate()
endpoint = self.endpoint_start + 'search/jobs/' + search_id
data = {'output_mode': self.output_mode}
return self.client.call_api(endpoint, 'GET', data=data)
def get_search_results(self, search_id, offset, count):
# sends a GET request to
# https://<server_ip>:<port>/services/search/jobs/<search_id>/results
# returns results associated with the search job.
self.authenticate()
endpoint = self.endpoint_start + "search/jobs/" + search_id + '/results'
data = {'output_mode': self.output_mode}
if ((offset is not None) and (count is not None)):
data['offset'] = str(offset)
data['count'] = str(count)
# response object body should contain information pertaining to search.
return self.client.call_api(endpoint, 'GET', urldata=data)
def delete_search(self, search_id):
# sends a DELETE request to
# https://<server_ip>:<port>/services/search/jobs/<search_id>
# cancels and deletes search created earlier.
self.authenticate()
endpoint = self.endpoint_start + 'search/jobs/' + search_id
data = {'output_mode': self.output_mode}
return self.client.call_api(endpoint, 'DELETE', data=data)
class APIClient:
def __init__(self, connection, configuration):
self.logger = logger.set_logger(__name__)
self.endpoint_start = ''
headers = dict()
host_port = connection.get('host') + ':' + str(
connection.get('port', ''))
headers['accept'] = 'application/json'
auth = configuration.get('auth')
if auth is not None and auth.get('token', None) is not None:
headers['Authorization'] = 'token {}'.format(auth.get('token'))
url_modifier_function = None
headers['user-agent'] = _USER_AGENT
self.timeout = connection['options'].get('timeout')
self.result_limit = connection['options'].get('result_limit')
if self.result_limit > _MAX_RESULT:
self.logger.warning(
"The length exceeds length limit. Use default length: %s",
_MAX_RESULT)
self.result_limit = _MAX_RESULT
self.client = RestApiClient(host_port,
None,
headers,
url_modifier_function,
cert_verify=connection.get(
'selfSignedCert', True),
sni=connection.get('sni', None))
def ping_data_source(self):
# Pings the data source
endpoint = 'tide/api/data/threats/state'
# now = datetime.datetime.utcnow().isoformat(timespec="milliseconds") + "Z"
# https://csp.infoblox.com:443/tide/api/data/threats/state?type=host&rlimit=1
return self.client.call_api(endpoint,
'GET',
timeout=self.timeout,
urldata={
"type": "host",
"rlimit": "1"
})
def get_search_results(self, search_id, range_start=None, range_end=None):
# Return the search results. Results must be in JSON format before being translated into STIX
# endpoint = self.endpoint_start + '/api/dnsdata/v2/dns_event'
payload = json.loads(search_id)
if payload['source'] == 'dnsEventData':
return self._get_dnseventdata_results(search_id, range_start,
range_end)
elif payload['source'] == 'dossierData':
return self._get_dossierdata_results(search_id, range_start,
range_end)
elif payload['source'] == 'tideDbData':
return self._get_tidedbdata_results(search_id, range_start,
range_end)
# default behavior
raise RuntimeError("Unknown source provided source={}".format(
payload['source']))
def _get_dnseventdata_results(self,
search_id,
range_start=None,
range_end=None):
endpoint = 'api/dnsdata/v2/dns_event'
headers = dict()
headers['Content-Type'] = 'application/json'
headers['Accept'] = 'application/json'
payload = json.loads(search_id)
resp_dict = dict()
resp_dict["data"] = []
start = range_start if range_start else 0
end = range_end if range_end else 0
offset = start
max_fetch_count = 10
for fetch_iteration in range(0, max_fetch_count):
params = {"_limit": self.result_limit, "_offset": offset}
resp = self.client.call_api(endpoint + "?" + payload["query"],
'GET',
urldata=params,
headers=headers,
timeout=self.timeout)
resp_dict["code"] = resp.code
if resp.code != 200:
if resp.code == 401:
resp_dict["message"] = resp.read().decode("utf-8")
else:
response_payload = json.loads(resp.read())
resp_dict["message"] = "\n".join([
error["message"] for error in response_payload["error"]
])
del resp_dict["data"]
return resp_dict
# successful request, append data to collection and recalculate offset
response_payload = json.loads(resp.read())
if "result" not in response_payload or len(
response_payload["result"]) == 0:
self.logger.debug("No additional results found")
break
offset += len(response_payload["result"])
for event in response_payload["result"]:
resp_dict["data"].append({"dnsEventData": event})
if len(resp_dict["data"]) > end - start:
resp_dict["data"] = resp_dict["data"][0:end - start]
break
if fetch_iteration == max_fetch_count - 1:
self.logger.warning("Reach max fetch count %s, stop loop",
max_fetch_count)
break
if resp_dict.get("code") == 200:
self.logger.debug("The DNS Event count is %s",
len(resp_dict["data"]))
return resp_dict
def _get_dossierdata_results(self,
search_id,
range_start=0,
range_end=None):
endpoint = 'tide/api/services/intel/lookup/indicator'
headers = dict()
headers['Content-Type'] = 'application/json'
headers['Accept'] = 'application/json'
payload = json.loads(search_id)
resp_dict = dict()
resp_dict["data"] = []
start = range_start if range_start else 0
end = range_end if range_end else 0
params = {'wait': 'true', 'source': 'pdns'}
# NOTE: Dossier does not support pagination via multiple requests. All results returned in the response.
resp = self.client.call_api(endpoint + "/" + payload["threat_type"] +
"?" + payload["query"],
'GET',
urldata=params,
headers=headers,
timeout=self.timeout)
resp_dict["code"] = resp.code
if resp.code != 200:
if resp.code == 401:
resp_dict["message"] = resp.read().decode("utf-8")
else:
response_payload = json.loads(resp.read())
resp_dict["message"] = response_payload["error"]
del resp_dict["data"]
return resp_dict
response_payload = json.loads(resp.read())
for i in response_payload["results"]:
for j in i["data"]["items"]:
restructure_payload = {
'job': {
'create_time': response_payload['job']['create_time']
},
'results': [{
'data': {
'items': [j]
}
}]
}
resp_dict["data"].append({"dossierData": restructure_payload})
# Trim result set based on min/max range values
end = end if end < len(resp_dict["data"]) else len(resp_dict["data"])
num_results = end - start
if len(resp_dict["data"]) > end - start:
resp_dict["data"] = resp_dict["data"][start:end]
if resp_dict.get("code") == 200:
self.logger.debug("The Dossier count is %s",
len(resp_dict["data"]))
return resp_dict
def _get_tidedbdata_results(self,
search_id,
range_start=0,
range_end=None):
endpoint = 'tide/api/data/threats/state'
headers = dict()
headers['Content-Type'] = 'application/json'
headers['Accept'] = 'application/json'
payload = json.loads(search_id)
resp_dict = dict()
resp_dict["data"] = []
start = range_start if range_start else 0
end = range_end if range_end else 0
params = {
"rlimit": self.result_limit,
}
if "type=" not in search_id:
params["type"] = payload["threat_type"]
if payload["threat_type"] == "ip":
params["include_ipv6"] = "true"
# NOTE: Tide does not support pagination via multiple requests. All results returned in the response.
resp = self.client.call_api(endpoint + "?" + payload["query"],
'GET',
urldata=params,
headers=headers,
timeout=self.timeout)
resp_dict["code"] = resp.code
if resp.code != 200:
if resp.code == 401:
resp_dict["message"] = resp.read().decode("utf-8")
else:
response_payload = json.loads(resp.read())
resp_dict["message"] = response_payload["error"]
del resp_dict["data"]
return resp_dict
response_payload = json.loads(resp.read())
for i in response_payload["threat"]:
resp_dict["data"].append({"tideDbData": i})
# Trim result set based on min/max range values
end = end if end < len(resp_dict["data"]) else len(resp_dict["data"])
num_results = end - start
if len(resp_dict["data"]) > end - start:
resp_dict["data"] = resp_dict["data"][start:end]
if resp_dict.get("code") == 200:
self.logger.debug("The TIDE count is %s", len(resp_dict["data"]))
return resp_dict
class APIClient():
AUTH_ENDPOINT = "rqt-api/1/authenticate"
EVENT_ENDPOINT = "rqt-api/1/events/hunt"
def __init__(self, connection, configuration):
self.headers = dict()
self.headers['Accept'] = 'application/json'
url_modifier_function = None
auth = configuration.get('auth')
self.host = connection.get('host')
self.client = RestApiClient(
connection.get('host'),
None,
self.headers,
url_modifier_function=url_modifier_function,
cert_verify=connection.get('selfSignedCert', True),
sni=connection.get('sni', None))
self.timeout = connection['options'].get('timeout')
self.app_id = auth['app_id']
self.secret_key = auth['secret_key']
self.token = None
self.token_expiresat = None
def ping_data_source(self):
# Pings the data source
return self.get_token()
def get_search_results(self, search_id, length):
# Return the search results. Results must be in JSON format before being translated into STIX
params = dict()
params['count'] = length
token_response = self.get_token()
response_code = token_response['code']
if response_code == 200:
self.headers['Authorization'] = 'Bearer {}'.format(
token_response['token'])
else:
raise Exception(token_response)
body_data = {'query': search_id}
return self.client.call_api(self.EVENT_ENDPOINT,
'POST',
urldata=params,
headers=self.headers,
data=body_data)
def page_search(self, search_id, next_page_url, length):
params = dict()
params['count'] = length
if not self.token_expired():
token_response = self.get_token()
response_code = token_response['code']
if response_code == 200:
self.headers['Authorization'] = 'Bearer {}'.format(
token_response['token'])
body_data = {'query': search_id}
page = next_page_url.split('?', maxsplit=1)[1]
next_page_endpoint = self.EVENT_ENDPOINT + '?' + page
return self.client.call_api(next_page_endpoint,
'POST',
headers=self.headers,
data=body_data)
def get_token(self):
auth_data = dict()
response_dict = dict()
auth_data['id'] = self.app_id
auth_data['secret'] = self.secret_key
try:
response = self.client.call_api(self.AUTH_ENDPOINT,
'POST',
headers=self.headers,
data=auth_data)
response_dict['code'] = response.code
response_text = json.loads(response.read())
if response.code == 200:
response_dict['token'] = response_text['token']
self.token_expiresat = response_text['expiresAt']
else:
response_dict[
'message'] = 'Authentication Error: Token Generation Failed. ' + response_text[
'message']
except Exception as ex:
if ex.__class__.__name__ == 'ConnectionError':
raise ConnectionError('Token Generation Failed: ' + str(ex))
else:
raise ex
return response_dict
def token_expired(self) -> bool:
"""Check if the token is expired.
:return: True if token is expired, False if not expired
:rtype: bool
"""
expires_at = datetime.fromtimestamp(self.token_expiresat)
return expires_at >= datetime.now()
class APIClient:
INCIDENTS_IDS_ENDPOINT = 'detects/queries/detects/v1'
INCIDENTS_INFO_ENDPOINT = 'detects/entities/summaries/GET/v1'
TOKEN_ENDPOINT = 'https://api.crowdstrike.com/oauth2/token'
"""API Client to handle all calls."""
def __init__(self, connection, configuration):
"""Initialization.
:param connection: dict, connection dict
:param configuration: dict,config dict"""
headers = dict()
url_modifier_function = None
auth = configuration.get('auth')
# self.endpoint_start = 'incidents/'
self.host = connection.get('host')
self.client = RestApiClient(connection.get('host'),
connection.get('port', None),
headers,
url_modifier_function=url_modifier_function,
cert_verify=connection.get('selfSignedCert', True),
sni=connection.get('sni', None)
)
self.timeout = connection['options'].get('timeout')
self._client_id = auth['client_id']
self._client_secret = auth['client_secret']
self._token = None
self._token_time = None
def get_detections_IDs(self, filter, limit, sort=None):
"""get the response from MSatp endpoints
:param filter: filter incidents by certain value
:param sort: sort incidents according to sort value
:return: response, json object"""
headers = dict()
data = dict()
headers['Content-Type'] = 'application/json'
headers['Accept'] = 'application/json'
headers['Authorization'] = f'Bearer {self.get_token()}'
endpoint = self.INCIDENTS_IDS_ENDPOINT
data['filter'] = filter
data['limit'] = limit
if sort:
data['sort'] = sort
return self.client.call_api(endpoint, 'GET', headers=headers, urldata=data, timeout=self.timeout)
def ping_box(self):
# Sends a GET request
headers = dict()
headers['Authorization'] = f'Bearer {self.get_token()}'
endpoint = 'detects/queries/detects/v1' # Test if system alive
return self.client.call_api(endpoint, 'GET', headers=headers, timeout=self.timeout)
def get_detections_info(self, ids):
"""get the response from crowdstrike endpoints
:param ids: Provide one or more incident IDs
:return: response, json object"""
headers = dict()
headers['Content-Type'] = 'application/json'
headers['Accept'] = 'application/json'
headers['Authorization'] = f'Bearer {self.get_token()}'
endpoint = self.INCIDENTS_INFO_ENDPOINT
ids_expression = json.dumps({'ids': ids}).encode("utf-8")
return self.client.call_api(endpoint, 'POST', headers=headers, data=ids_expression, timeout=self.timeout)
def get_token(self) -> str:
"""Request a new OAuth2 token.
:return: [description]
:rtype: str
"""
if self.token_expired():
resp = requests.request(
'POST',
self.TOKEN_ENDPOINT,
headers={
'accept': 'application/json',
'Content-Type': 'application/x-www-form-urlencoded'
},
data=(
f'client_id={self._client_id}'
f'&client_secret={self._client_secret}'
)
)
token = resp.json().get('access_token')
self._token = token
self._token_time = datetime.now()
return self._token
def token_expired(self) -> bool:
"""Check if the OAuth2 token is expired.
:return: True if token is expired, False if not expired
:rtype: bool
"""
expired = True
if self._token:
expired = (datetime.now() - self._token_time) >= timedelta(minutes=30)
return expired
class APIClient:
LOGIN_ENDPOINT = 'login.html'
QUERY_ENDPOINT = "rest/visualsearch/query/simple"
LOGOFF_ENDPOINT = 'logout'
def __init__(self, connection, configuration):
self.logger = logger.set_logger(__name__)
headers = {'Content-Type': 'application/json'}
url_modifier_function = None
self.auth = configuration.get('auth')
self.client = RestApiClient(
connection.get('host'),
connection.get('port', None),
headers,
url_modifier_function=url_modifier_function)
self.timeout = connection['options'].get('timeout')
def ping_box(self):
"""
Ping the Data Source
:return: Response object
"""
headers = {}
query = '{"queryPath": [{"requestedType": "Connection", "filters": [], "isResult": true}],' \
'"totalResultLimit": 1, "perGroupLimit": 1,"templateContext": "SPECIFIC"}'
headers['Cookie'] = self.session_log_in()
return self.client.call_api(self.QUERY_ENDPOINT,
'POST',
headers=headers,
data=query)
def get_search_results(self, query):
"""
Get results from Data Source
:param query: Data Source Query
:return: Response Object
"""
headers = {'Cookie': self.session_log_in()}
self.logger.debug("query: %s", query)
return self.client.call_api(self.QUERY_ENDPOINT,
'POST',
headers=headers,
data=query)
def session_log_in(self):
"""
Create a login session and return the cookie id
:return: str, cookie id
"""
headers = {"Content-Type": "application/x-www-form-urlencoded"}
response_wrapper = self.client.call_api(self.LOGIN_ENDPOINT,
'POST',
headers=headers,
data=self.auth)
cookie_id = response_wrapper.response.request.headers
return cookie_id['Cookie']
def session_log_out(self, response_wrapper):
"""
Logging out of Session
:return: response object
"""
headers = {}
cookie_dict = response_wrapper.response.request.headers
cookie_id = cookie_dict["Cookie"]
headers["Cookie"] = cookie_id
return self.client.call_api(self.LOGOFF_ENDPOINT,
'GET',
headers=headers)
class APIClient():
# API METHODS
# These methods are used to call Ariel's API methods through http requests.
# Each method makes use of the http methods below to perform the requests.
# This class will encode any data or query parameters which will then be
# sent to the call_api() method of the RestApiClient
PING_TIMEOUT_IN_SECONDS = 10
def __init__(self, connection, configuration):
# This version of the ariel APIClient is designed to function with
# version 6.0 of the ariel API.
self.endpoint_start = 'api/ariel/'
self.urldata = {}
headers = dict()
host_port = connection.get('host') + ':' + \
str(connection.get('port', ''))
headers['version'] = '8.0'
headers['accept'] = 'application/json'
auth = configuration.get('auth')
if auth != None and auth.get('SEC', None) != None:
headers['sec'] = auth.get('SEC')
url_modifier_function = None
proxy = connection.get('proxy')
if proxy is not None:
proxy_url = proxy.get('url')
proxy_auth = proxy.get('auth')
if (proxy_url is not None and proxy_auth is not None):
headers['proxy'] = proxy_url
headers['proxy-authorization'] = 'Basic ' + proxy_auth
if proxy.get('x_forward_proxy', None) is not None:
headers['x-forward-url'] = 'https://' + \
host_port + '/' # + endpoint, is set by 'add_endpoint_to_url_header'
host_port = proxy.get('x_forward_proxy')
if proxy.get('x_forward_proxy_auth', None) is not None:
headers['x-forward-auth'] = proxy.get(
'x_forward_proxy_auth')
headers['user-agent'] = 'UDS'
url_modifier_function = self.add_endpoint_to_url_header
self.data_lake = connection.get('data_lake')
if self.data_lake:
print('QRadar Cloud Data Lake enabled')
self.client = RestApiClient(
host_port,
None,
connection.get('cert', None),
headers,
url_modifier_function,
cert_verify=connection.get('selfSignedCert', True),
mutual_auth=connection.get('use_securegateway', False),
sni=connection.get('sni', None))
def add_endpoint_to_url_header(self, url, endpoint, headers):
# this function is called from 'call_api' with proxy forwarding,
# it concatenates the endpoint to the header containing the url.
headers['x-forward-url'] += endpoint
# url is returned since it points to the proxy for initial call
return url
def ping_box(self):
# Sends a GET request
# to https://<server_ip>/api/help/resources
endpoint = 'api/help/resources' # no 'ariel' in the path
return self.client.call_api(endpoint,
'GET',
timeout=self.PING_TIMEOUT_IN_SECONDS)
def get_databases(self):
# Sends a GET request
# to https://<server_ip>/api/ariel/databases
endpoint = self.endpoint_start + 'databases'
return self.client.call_api(endpoint, 'GET')
def get_database(self, database_name):
# Sends a GET request
# to https://<server_ip>/api/ariel/databases/<database_name>
endpoint = self.endpoint_start + 'databases' + '/' + database_name
return self.client.call_api(endpoint, 'GET')
def get_searches(self):
# Sends a GET request
# to https://<server_ip>/api/ariel/searches
endpoint = self.endpoint_start + "searches"
# Send requests to QRadar Cloud Data Lake
if self.data_lake:
self.urldata.update({'data_lake': '"qcdl"'})
return self.client.call_api(endpoint, 'GET', urldata=self.urldata)
def create_search(self, query_expression):
# Sends a POST request
# to https://<server_ip>/api/ariel/searches
endpoint = self.endpoint_start + "searches"
data = {'query_expression': query_expression}
# Send requests to QRadar Cloud Data Lake
if self.data_lake:
self.urldata.update({'data_lake': '"qcdl"'})
return self.client.call_api(endpoint,
'POST',
data=data,
urldata=self.urldata)
def get_search(self, search_id):
# Sends a GET request to
# https://<server_ip>/api/ariel/searches/<search_id>
endpoint = self.endpoint_start + "searches/" + search_id
# Send requests to QRadar Cloud Data Lake
if self.data_lake:
self.urldata.update({'data_lake': '"qcdl"'})
return self.client.call_api(endpoint, 'GET', urldata=self.urldata)
def get_search_results(self,
search_id,
response_type,
range_start=None,
range_end=None):
# Sends a GET request to
# https://<server_ip>/api/ariel/searches/<search_id>
# response object body should contain information pertaining to search.
headers = dict()
headers['Accept'] = response_type
if ((range_start is not None) and (range_end is not None)):
headers['Range'] = ('items=' + str(range_start) + '-' +
str(range_end))
endpoint = self.endpoint_start + "searches/" + search_id + '/results'
# Send requests to QRadar Cloud Data Lake
if self.data_lake:
self.urldata.update({'data_lake': '"qcdl"'})
return self.client.call_api(endpoint,
'GET',
headers,
urldata=self.urldata)
def update_search(self, search_id, save_results=None, status=None):
# Sends a POST request to
# https://<server_ip>/api/ariel/searches/<search_id>
# posts search result to site
endpoint = self.endpoint_start + "searches/" + search_id
data = {}
if save_results:
data['save_results'] = save_results
if status:
data['status'] = status
# Send requests to QRadar Cloud Data Lake
if self.data_lake:
self.urldata.update({'data_lake': '"qcdl"'})
return self.client.call_api(endpoint,
'POST',
data=data,
urldata=self.urldata)
def delete_search(self, search_id):
# Sends a DELETE request to
# https://<server_ip>/api/ariel/searches/<search_id>
# deletes search created earlier.
endpoint = self.endpoint_start + "searches" + '/' + search_id
# Send requests to QRadar Cloud Data Lake
if self.data_lake:
self.urldata.update({'data_lake': '"qcdl"'})
return self.client.call_api(endpoint, 'DELETE', urldata=self.urldata)
class GuardApiClient(RestApiClient):
def __init__(self, params, host, port, headers, url_modifier_function,
cert_verify, sni, auth):
self.client = RestApiClient(host, port, headers, url_modifier_function,
cert_verify, sni, auth)
self.logger = logger.set_logger(__name__)
self.url = params["url"]
self.secret = params["client_secret"]
self.user = params["config_uname"]
self.password = params["config_pass"]
self.client_id = params["client_id"]
self.token_target = 'oauth/token'
self.report_target = 'restAPI/online_report'
self.qs_target = 'restAPI/quick_search'
self.fields_target = 'restAPI/fieldsTitles'
self.fields = {}
self.get_token()
# -------------------------------------------------------------------------------
# REPORT parameters
# -------------------------------------------------------------------------------
# TBD dates
# self.set_call_dates()
# self.QUERY_FROM_DATE = "Now -60 DAY"
# self.QUERY_TO_DATE = "Now"
# -------------------------------------------------------------------------------
# QS parameters
# -------------------------------------------------------------------------------
# TBD dates
# self.qs_startTime = "20200616 10:00:00"
# self.qs_endTime = "20200616 21:00:00"
def set_call_dates(self):
# look for last_run file - if file exist read last run date from it
# if file does not exist or date older then x days set from dates to now -1 Day
# otherwise set from date = last run date
self.now = datetime.datetime.now()
self.qs_endTime = self.now.strftime("%Y-%m-%d %H:%M:%S")
self.QUERY_TO_DATE = self.now.strftime("%Y-%m-%d %H:%M:%S")
from_file = None
try:
file = open("./last_run", "r")
try:
text = file.read()
from_file = json.loads(text)
finally:
file.close()
except:
pass
if from_file and self.url in from_file:
period_start = datetime.datetime.strptime(from_file[self.url],
"%Y/%m/%d %H:%M:%S")
if self.now - period_start > datetime.timedelta(days=2):
period_start = self.now - datetime.timedelta(days=1)
else:
period_start = self.now - datetime.timedelta(days=1)
self.from_file = from_file
self.QUERY_FROM_DATE = period_start.strftime("%Y-%m-%d %H:%M:%S")
self.qs_startTime = period_start.strftime("%Y-%m-%d %H:%M:%S")
def save_last_run_date(self):
try:
if self.from_file:
output = self.from_file
output[self.url] = self.now.strftime("%Y/%m/%d %H:%M:%S")
else:
output = {self.url: self.now.strftime("%Y/%m/%d %H:%M:%S")}
file = open("./last_run", "w")
file.write(json.dumps(output))
finally:
file.close()
def get_token(self):
# -------------------------------------------------------------------------------
# Authentication
# -------------------------------------------------------------------------------
# comment in and out all prints
# print("client_id="+self.client_id)
# print("secret="+self.secret)
# print("user="******"password="******"token ", True):
self.access_token = json.loads(response.read())['access_token']
# print("token="+ self.access_token)
self.headers = {
'Content-Type': 'application/json',
'Authorization': 'Bearer {0}'.format(self.access_token)
}
def request_token(self):
self.token_data = 'client_id={0}&grant_type=password&client_secret={1}&username={2}&password={3}'.format(
self.client_id, self.secret, self.user, self.password)
return self.client.call_api(self.token_target,
'POST',
urldata=self.token_data)
def validate_response(self, p_response, prefix, abort=False):
if p_response.code != 200:
if abort:
raise Exception(prefix + "request faild " +
str(p_response.code) + "-" + p_response.read())
return False
return True
def handle_report(self, params, index_from, fetch_size):
# -------------------------------------------------------------------------------
# REPORT
# -------------------------------------------------------------------------------
results = ""
# context().logger.debug('------------------- ' + report_name + ' ----------------------')
params["fetchSize"] = int(fetch_size)
params["indexFrom"] = int(index_from)
params["inputTZ"] = "UTC"
rest_data = json.dumps(params)
response = self.client.call_api(self.report_target,
'POST',
headers=self.headers,
data=rest_data)
results = response.read()
if not isinstance(results, list):
try:
# errorCode = results["ErrorCode"]
errorCode = response.code
# For compatibility with Guardium -
# inputTZ parameter was added after v11.3
# so in case it does not exist execute the query without it
if errorCode == "27":
params.pop("inputTZ")
rest_data = json.dumps(params)
self.logger.warn(
"InputTZ not suppoerted - running query without it")
response = self.client.call_api(self.report_target,
'POST',
headers=self.headers,
data=rest_data)
except:
pass
return response
def handle_qs(self, params, index_from, fetch_size):
# -------------------------------------------------------------------------------
# QS
# -------------------------------------------------------------------------------
if not self.fields:
self.get_field_titles()
results = ""
params["fetchSize"] = int(fetch_size - 1)
params["firstPosition"] = int(index_from - 1)
params["inputTZ"] = "UTC"
rest_data = json.dumps(params)
response = self.client.call_api(self.qs_target,
'POST',
data=rest_data,
headers=self.headers)
results = response.read()
if not isinstance(results, list):
try:
errorCode = response.code
# For compatibility with Guardium -
# inputTZ parameter was added after v11.3
# so in case it does not exist execute the query without it
if errorCode == "27":
params.pop("inputTZ")
rest_data = json.dumps(params)
self.logger.warn(
"InputTZ not suppoerted - running query without it")
response = self.client.call_api(self.qs_target,
'POST',
data=rest_data,
headers=self.headers)
except:
pass
response.content = self.translate_response(json.loads(self.fields),
json.loads(response.read()))
return response
def get_field_titles(self):
# get QS field titles from Guardium
response = self.client.call_api(self.fields_target,
'GET',
headers=self.headers)
# response = requests.get(self.url + self.fields_target, headers=self.headers, verify=False)
try:
msg = json.loads(response.read())["Message"]
except Exception as e:
self.fields = json.dumps(json.loads(response.read())[0])
return
self.fields = msg
def translate_response(self, fields, results):
# translate fields from numeric tags to field titles
# set to lower case, replace white spaces with _
res = []
for result in results:
num_rows = result["numRows"]
count = result["count"]
category = result["searchArgs"]["category"]
# print("total num rows " + str(num_rows) + " count " + str(count))
if num_rows > 0:
res = []
items = result["items"]
# print(items)
i = 0
for item in items:
res_item = {}
for key, value in fields.items():
try:
val = key.split(";")
if item.get(val[0]) is None:
continue
if len(val) > 1:
item_value = ""
for val1 in val:
item_value = item_value + str(
item[val1]) + " "
item_value = item_value.rstrip()
else:
item_value = item[key]
value = value.lower().replace(" ", "_")
if value == "date_time":
value = "timestamp"
res_item[value] = item_value
# print(str(value)+ '->'+str(res_item[value]))
except Exception as e:
print("ERROR: Category: " + category + " key: " +
key + " value: " + value)
print(e)
res.append(res_item)
return json.dumps(res)
class APIClient:
def __init__(self, connection, configuration):
self.logger = logger.set_logger(__name__)
self.endpoint_start = 'v2.0/xdr/'
headers = dict()
host_port = connection.get('host') + ':' + str(
connection.get('port', ''))
headers['accept'] = 'application/json'
auth = configuration.get('auth')
if auth is not None and auth.get('token', None) is not None:
headers['Authorization'] = 'Bearer {}'.format(auth.get('token'))
url_modifier_function = None
headers['user-agent'] = _USER_AGENT
self.timeout = connection['options'].get('timeout')
self.client = RestApiClient(host_port,
None,
headers,
url_modifier_function,
cert_verify=connection.get(
'selfSignedCert', True),
sni=connection.get('sni', None))
def ping_data_source(self):
# Pings the data source
endpoint = 'v2.0/siem/events'
now = datetime.datetime.utcnow().isoformat(
timespec="milliseconds") + "Z"
resp = self.client.call_api(endpoint,
'GET',
timeout=self.timeout,
urldata={
"startDateTime": now,
"endDateTime": now
})
return {"code": resp.code}
def get_search_results(self, search_id, range_start=None, range_end=None):
# Return the search results. Results must be in JSON format before being translated into STIX
endpoint = self.endpoint_start + 'search/data'
headers = dict()
headers['Content-Type'] = 'application/json'
headers['Accept'] = 'application/json'
payload = json.loads(search_id)
resp_dict = dict()
all_data = list()
resp_dict["data"] = {"logs": all_data}
start = range_start if range_start else 0
end = range_end if range_end else 0
offset = start
max_fetch_count = 10
for i in range(0, max_fetch_count):
code, response = self._fetch(endpoint, headers, payload, offset)
resp_dict["code"] = code
if code == 200:
logs = response["data"]['logs']
if "offset" not in response["data"]:
# not support offset
if len(logs) > range_start:
all_data += logs[range_start:range_end]
break
if not logs:
break
elif end and len(all_data) + len(logs) >= end - start:
offset += len(logs)
all_data += logs[0:end - start - len(all_data)]
break
else:
offset += len(logs)
all_data += logs
else:
resp_dict["message"] = response["error"]["message"]
del resp_dict["data"]
break
if i == max_fetch_count - 1:
self.logger.warning("Reach max fetch count %s, stop loop",
max_fetch_count)
if resp_dict.get("code") == 200:
self.logger.debug("The log count is %s",
len(resp_dict["data"]["logs"]))
return resp_dict
def _fetch(self, endpoint, headers, payload, offset):
payload["offset"] = offset
resp = self.client.call_api(endpoint,
'POST',
headers=headers,
data=json.dumps(payload),
timeout=self.timeout)
payload_dict = json.loads(resp.read())
return resp.code, payload_dict
class APIClient:
TOKEN_ENDPOINT = 'core-service/rest/LoginService/login'
STATUS_ENDPOINT = 'server/search/status'
QUERY_ENDPOINT = 'server/search'
RESULT_ENDPOINT = 'server/search/events'
DELETE_ENDPOINT = 'server/search/close'
def __init__(self, connection, configuration):
self.connector = __name__.split('.')[1]
self.auth = configuration.get('auth')
headers = {'Accept': 'application/json'}
self.client = RestApiClient(connection.get('host'),
connection.get('port'),
headers,
cert_verify=connection.get('selfSignedCert', True)
)
def ping_data_source(self):
data, headers = dict(), dict()
data['search_session_id'] = int(round(time.time() * 1000))
data['user_session_id'] = self.get_user_session_id()
data['start_time'] = self.get_current_time()['start_time']
data['end_time'] = self.get_current_time()['end_time']
headers['Content-Type'] = 'application/json'
headers['Accept-Charset'] = 'utf-8'
return self.client.call_api(self.QUERY_ENDPOINT, 'POST', headers, data=json.dumps(data))
def create_search(self, query_expression):
return_obj = dict()
auth = dict()
auth['search_session_id'] = int(round(time.time() * 1000))
auth['user_session_id'] = self.get_user_session_id()
try:
query = json.loads(query_expression)
query.update(auth)
headers = {'Content-Type': 'application/json', 'Accept-Charset': 'utf-8'}
response = self.client.call_api(self.QUERY_ENDPOINT, 'POST', headers, data=json.dumps(query))
raw_response = response.read()
response_code = response.code
if 199 < response_code < 300:
response_dict = json.loads(raw_response)
if response_dict.get('sessionId'):
return_obj['success'] = True
return_obj['search_id'] = str(auth['search_session_id']) + ':' + str(auth['user_session_id'])
# arcsight logger error codes - currently unavailable state
elif response_code in [500, 503]:
response_string = raw_response.decode()
ErrorResponder.fill_error(return_obj, response_string, ['message'], connector=self.connector)
elif isinstance(json.loads(raw_response), dict):
response_error = json.loads(raw_response)
response_dict = response_error['errors'][0]
ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector)
else:
raise Exception(raw_response)
return return_obj
except Exception as err:
raise err
def get_search_status(self, search_session_id, user_session_id):
headers, params = dict(), dict()
params['search_session_id'] = int(search_session_id)
params['user_session_id'] = user_session_id
headers['Content-Type'] = 'application/json'
headers['Accept-Charset'] = 'utf-8'
return self.client.call_api(self.STATUS_ENDPOINT, 'POST', headers, data=json.dumps(params))
def get_search_results(self, search_session_id, user_session_id, range_start=None, range_end=None):
headers, params = dict(), dict()
params['search_session_id'] = int(search_session_id)
params['user_session_id'] = user_session_id
params['offset'] = int(range_start)
params['length'] = int(range_end)
headers['Content-Type'] = 'application/json'
headers['Accept-Charset'] = 'utf-8'
return self.client.call_api(self.RESULT_ENDPOINT, 'POST', headers, data=json.dumps(params))
def delete_search(self, search_session_id, user_session_id):
headers, params = dict(), dict()
params['search_session_id'] = int(search_session_id)
params['user_session_id'] = user_session_id
headers['Content-Type'] = 'application/json'
headers['Accept-Charset'] = 'utf-8'
return self.client.call_api(self.DELETE_ENDPOINT, 'POST', headers, data=json.dumps(params))
def get_user_session_id(self):
try:
response = self.client.call_api(self.TOKEN_ENDPOINT, 'POST', data=self.auth)
if response.code == 200:
response_text = json.loads(response.read())
token = response_text['log.loginResponse']['log.return']
elif response.read().decode("utf-8") == '':
return_dict = 'Request error or authentication failure.'
raise Exception(return_dict)
else:
raise Exception(response)
return token
except Exception as err:
raise err
@staticmethod
def get_current_time():
ping_time = dict()
end_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z'
start_time = (datetime.utcnow() - timedelta(minutes=5)).strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z'
ping_time['start_time'] = start_time
ping_time['end_time'] = end_time
return ping_time
class APIClient:
endpoint_start = '/v1.0/events'
toeken_endpoint = '/v1.0/endpoint/default/token'
def __init__(self, connection, configuration):
self.logger = logger.set_logger(__name__)
headers = dict()
url_modifier_function = None
auth = configuration.get('auth')
# self.endpoint_start = 'incidents/'
self.host = connection.get('host')
self.client = RestApiClient(
connection.get('host'),
connection.get('port', None),
headers,
url_modifier_function=url_modifier_function,
cert_verify=connection.get('selfSignedCert', False),
sni=connection.get('sni', None))
self.timeout = connection['options'].get('timeout')
self._client_id = auth['clientId']
self._client_secret = auth['clientSecret']
self._token = None
self._token_time = None
def get_token(self):
"""get the token and if expired re-generate and store in token variable"""
tokenResponse = self.generate_token()
return tokenResponse.json().get('access_token')
def generate_token(self):
"""To generate the Token"""
if self.token_expired():
resp = requests.request(
'POST',
'https://' + self.host + self.toeken_endpoint,
headers={
'accept': 'application/json',
'Content-Type': 'application/x-www-form-urlencoded'
},
data=(f'client_id={self._client_id}'
f'&client_secret={self._client_secret}'
f'&grant_type=client_credentials'
f'&scope=openid'))
token = resp.json().get('access_token')
self._token = token
self._token_time = datetime.now()
self.resp = resp
return self.resp
def token_expired(self) -> bool:
"""Check if the verify token is expired.
:return: True if token is expired, False if not expired
:return type: bool
"""
expired = True
if self._token:
expired = (datetime.now() -
self._token_time) >= timedelta(minutes=30)
return expired
def run_search(self, query_expr, range_end=None):
"""get the response from verify endpoints
:param quary_expr: dict, filter parameters
:param range_end: int,length value
:return: response, json object"""
events = []
# if self._token :
events = self.get_events(query_expr)
return self.response_handler(events, query_expr)
def get_events(self, query_expr):
self.headers = {
'Content-Type': 'application/json',
'Authorization': 'Bearer {0}'.format(self.get_token())
}
if query_expr is None:
data = None
return self.client.call_api(self.endpoint_start,
'GET',
self.headers,
urldata=query_expr)
def response_handler(self, data=None, query_expr=None):
if data is None:
data = []
response = dict()
response['data'] = json.loads(data.read())
response['error'] = data.code
response['code'] = data.code
response['error_msg'] = data.response.reason
response['success'] = data.code
if response['code'] == 200:
response['search_after'] = response.get(
"data")['response']['events']['search_after']
try:
response['event_data'] = self.parseJson(
response.get("data")['response']['events']['events'])
except KeyError:
self.logger.debug('events data not found in respose object',
response)
response['event_data'] = []
elif response['error'] == 500 and "true" in response['error_msg']:
response.update({"code": 200, "data": []})
else:
response["message"] = data.response.reason
return response
def parseJson(self, response):
'''
Iterate through the response and read the nested data like geoip and data object.
'''
jsonObj = response
finalJson = dict()
parsedJson = []
for obj in jsonObj:
dictC = obj
if "geoip" in dictC:
dictA = json.loads(json.dumps(obj["geoip"]))
del dictC["geoip"]
dictB = json.loads(json.dumps(obj["data"]))
del dictC["data"]
dict_geo_location = json.loads(
json.dumps(dictA.get('location')))
del dictA['location']
finalJson = {**dictA, **dictB, **dict_geo_location}
else:
dictB = json.loads(json.dumps(obj["data"]))
del dictC["data"]
finalJson = dictB
remainingJson = json.loads(json.dumps(dictC))
finalJson = {**finalJson, **remainingJson}
parsedJson.append(finalJson)
return parsedJson
def key_exist(self, data_element, *keysarr):
'''
Check if *keys (nested) exists in `element` (dict).
'''
if not isinstance(data_element, dict):
raise AttributeError(
'keys_exists() expects dict as first argument.')
if len(keysarr) == 0:
raise AttributeError(
'keys_exists() expects at least two arguments, one given.')
_element = data_element
for key in list(keysarr):
try:
_element = _element[key]
except KeyError:
self.logger.debug('key not found ', key)
return False
return True
def get_search_results(self,
search_id,
response_type,
range_start=None,
range_end=None):
# Sends a GET request to
# https://<server_ip>//<search_id>
# response object body should contain information pertaining to search.
#https://isrras.ice.ibmcloud.com/v1.0/events?event_type="sso"&size=10&after="1640104162523","eeb40fd5-6b84-4dc9-9251-3f7a4cfd91c0"
headers = dict()
headers['Accept'] = response_type
size = 1000
if ((range_start is None) and (range_end is None)):
size = range_end - range_start
request_param = search_id + "& size=" + str(size)
endpoint = self.endpoint_start + request_param
return self.run_search(search_id)
def get_search(self, search_id):
# Sends a GET request to
# https://<server_ip>/api/ariel/searches/<search_id>
response = self.run_search(search_id)
return response
def delete_search(self, search_id):
return self.run_search(search_id)
