def create_results_connection(self, search_id, offset, length):
response = self.api_client.get_search_results(search_id, offset,
length)
response_code = response.code
response_text = response.read()
error = None
response_dict = dict()
try:
response_dict = json.loads(response_text)
except ValueError as ex:
self.logger.debug(response_text)
error = Exception(f'Can not parse response: {ex}')
return_obj = dict()
return_obj['success'] = False
if response_dict and response_code == 200:
return_obj['success'] = True
return_obj['data'] = response_dict['results']
else:
ErrorResponder.fill_error(return_obj,
response_dict, ['message'],
error=error,
connector=self.connector)
return return_obj
def create_results_connection(self, search_id, offset, length):
# Grab the response, extract the response code, and convert it to readable json
# Verify the input
response = self.api_client.get_search_results(search_id,
'application/json',
offset, length)
response_code = response.code
# Construct a response object
return_obj = dict()
results = json.loads(response.read())
if response_code == 200:
return_obj['success'] = True
# In the case of no results datasource returns a json/dict type reponse :
# {'ID': 0, 'Message': 'The Query did not retrieve any records'}
# Therefore setting empty list after checking the datatype
if isinstance(results, dict):
return_obj['data'] = []
else:
return_obj['data'] = results
return_obj["search_id"] = search_id
else:
ErrorResponder.fill_error(return_obj, results, ['message'])
return return_obj
def create_search(self, query_expression):
return_obj = dict()
auth = dict()
auth['search_session_id'] = int(round(time.time() * 1000))
auth['user_session_id'] = self.get_user_session_id()
try:
query = json.loads(query_expression)
query.update(auth)
headers = {'Content-Type': 'application/json', 'Accept-Charset': 'utf-8'}
response = self.client.call_api(self.QUERY_ENDPOINT, 'POST', headers, data=json.dumps(query))
raw_response = response.read()
response_code = response.code
if 199 < response_code < 300:
response_dict = json.loads(raw_response)
if response_dict.get('sessionId'):
return_obj['success'] = True
return_obj['search_id'] = str(auth['search_session_id']) + ':' + str(auth['user_session_id'])
# arcsight logger error codes - currently unavailable state
elif response_code in [500, 503]:
response_string = raw_response.decode()
ErrorResponder.fill_error(return_obj, response_string, ['message'], connector=self.connector)
elif isinstance(json.loads(raw_response), dict):
response_error = json.loads(raw_response)
response_dict = response_error['errors'][0]
ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector)
else:
raise Exception(raw_response)
return return_obj
except Exception as err:
raise err
def create_results_connection(self, search_id, offset, length):
try:
min_range = offset
max_range = offset + length
# Grab the response, extract the response code, and convert it to readable json
response = self.api_client.get_search_results(
search_id, min_range, max_range)
response_code = response.code
response_dict = json.loads(response.read())
# # Construct a response object
return_obj = dict()
if 200 <= response_code < 300:
return_obj['success'] = True
return_obj['data'] = response_dict
else:
ErrorResponder.fill_error(return_obj, response_dict,
['messages', 0, 'text'])
return return_obj
except Exception as err:
self.logger.error(
'error when getting search results: {}'.format(err))
import traceback
self.logger.error(traceback.print_stack())
raise
def ping_connection(self):
"""
Creates a synchronous ping connection.
NOTE: This checks if the Infoblox APIs are up and available.
:return: response object (includes success, code, and message fields)
:rtype: object
"""
try:
response = self.api_client.ping_data_source()
response_code = response.code
response_body = response.read().decode('utf-8')
# Construct a response object
return_obj = dict()
if response_code == 200:
return_obj['success'] = True
else:
response_dict = {
'code': response_code,
'message': response_body
}
ErrorResponder.fill_error(return_obj,
response_dict, ['message'],
connector=self.connector)
return return_obj
except Exception as err:
self.logger.error('error when pinging datasource: %s',
err,
exc_info=True)
raise
def create_results_connection(self, search_id, offset=None, length=None):
try:
# currently not implemented
# will implement later TODO
min_range = int(offset)
max_range = min_range + int(length)
response_dict = self.api_client.get_search_results(
search_id, min_range, max_range)
response_code = response_dict["code"]
return_obj = dict()
if response_code == 200:
return_obj['success'] = True
return_obj['data'] = response_dict['data']
else:
ErrorResponder.fill_error(return_obj, response_dict,
['message'])
return return_obj
except Exception as err:
print('error when getting search results: {}'.format(err))
import traceback
print(traceback.print_stack())
raise
def create_results_connection(self, search_id, offset, length):
try:
offset_i = int(offset)
len_i = int(length)
min_range = offset_i
if len_i > 1000:
self.logger.warning(
"The length exceeds length limit. Use default length: 1000"
)
max_range = offset_i + len_i if len_i <= 1000 else offset_i + 1000
# Grab the response, extract the response code, and convert it to readable json
response_dict = self.api_client.get_search_results(
search_id, min_range, max_range)
response_code = response_dict["code"]
# # Construct a response object
return_obj = dict()
if response_code == 200:
return_obj['success'] = True
return_obj['data'] = response_dict['data']['logs']
else:
ErrorResponder.fill_error(return_obj,
response_dict, ['message'],
connector=self.connector)
return return_obj
except Exception as err:
self.logger.error('error when getting search results: %s',
err,
exc_info=True)
raise
def get_events(self):
payload = "{\"name\": \"Secret Server Events Logs\", \"parameters\": [{\"name\": \"startDate\", \"value\": '%s'} , {\"name\":\"endDate\",\"value\": '%s'}]}" % (
self.startDate, self.endDate)
headers = {
'Authorization': self.accessToken,
'Content-Type': 'application/json'
}
endpoint = "SecretServer/api/v1/reports/execute"
response = RestApiClient.call_api(self,
endpoint,
'POST',
headers=headers,
data=payload,
urldata=None,
timeout=None)
return_obj = {}
if response.code != 200:
response_txt = response.response.text
ErrorResponder.fill_error(return_obj, message=response_txt)
raise Exception(return_obj)
collection = []
json_data = response.response.text
eventData = json.loads(json_data)
col = eventData['columns']
for obj in eventData['rows']:
obj = dict(zip(col, obj))
collection.append(obj)
return collection
def create_results_connection(self, query, offset, length):
""""built the response object
:param query: str, search_id
:param offset: int,offset value
:param length: int,length value"""
response_txt = None
return_obj = dict()
try:
if self.init_error is not None:
raise Exception(self.init_error)
response = self.api_client.run_search(query, offset, length)
return_obj = self._handle_errors(response, return_obj)
response_json = json.loads(return_obj["data"])
return_obj['data'] = response_json['Results']
# Customizing the output json,
# Get 'TableName' attribute from each row of event data
# Create a dictionary with 'TableName' as key and other attributes in an event data as value
# Filter the "None" and empty values except for RegistryValueName, which support empty string
# Customizing of Registryvalues json
table_event_data = []
for event_data in return_obj['data']:
lookup_table = event_data['TableName']
event_data.pop('TableName')
build_data = dict()
build_data[lookup_table] = {
k: v
for k, v in event_data.items()
if v or k == "RegistryValueName"
}
if lookup_table == "RegistryEvents":
registry_build_data = copy.deepcopy(build_data)
registry_build_data[lookup_table]["RegistryValues"] = []
registry_value_dict = {}
for k, v in build_data[lookup_table].items():
if k in [
"RegistryValueData", "RegistryValueName",
"RegistryValueType"
]:
registry_value_dict.update({k: v})
registry_build_data[lookup_table].pop(k)
registry_build_data[lookup_table]["RegistryValues"].append(
registry_value_dict)
build_data[lookup_table] = registry_build_data[
lookup_table]
build_data[lookup_table]['event_count'] = '1'
table_event_data.append(build_data)
return_obj['data'] = table_event_data
return return_obj
except Exception as ex:
if response_txt is not None:
ErrorResponder.fill_error(return_obj,
message='unexpected exception')
self.logger.error('can not parse response: ' +
str(response_txt))
else:
raise ex
def create_results_connection(self, searchID, offset, length):
params = {}
return_obj = {}
params["accountID"] = self.auth.get("accountID")
params["host"] = self.host
try:
params["accessToken"] = self.auth_token.obtainAccessToken()
except Exception as e:
ErrorResponder.fill_error(return_obj,
{'code': "Authorizaion Failed"},
message=str(e),
connector=self.connector)
return return_obj
try:
data = self.StixPatternProcessor.process(searchID, params)
return_obj["success"] = True
return_obj["data"] = data
return return_obj
except Exception as e:
ErrorResponder.fill_error(return_obj, {'code': "query_failed"},
message=str(e),
connector=self.connector)
return return_obj
def create_results_connection(self, search_id, offset, length):
offset = int(offset)
length = int(length)
try:
response = self.api_client.run_search(search_id, length)
response_code = response['code']
# Construct a response object
return_obj = dict()
if response_code == 200:
return_obj['success'] = True
return_obj['data'] = response.get("event_data", [])
return_obj['search_after'] = response.get("search_after", [])
# filter data based on filter_attr
# slice the records as per the provided offset and length(limit)
return_obj['data'] = return_obj['data'][offset:length]
else:
ErrorResponder.fill_error(return_obj, response, ['message'])
except Exception as err:
self.logger.error(
'error when getting search results: {}'.format(err))
self.logger.error(traceback.print_stack())
raise
return return_obj
def create_status_connection(self, search_id):
# Grab the response, extract the response code, and convert it to readable json
response = self.api_client.get_search(search_id)
response_code = response.code
response_dict = json.load(response)
status, progress = '', ''
if 'entry' in response_dict and isinstance(response_dict['entry'], list):
content = response_dict['entry'][0]['content']
progress = math.ceil(content['doneProgress'] * 100) # convert 0-1.0 scale to <int>0-100
status = content['dispatchState']
if status == StatusSplunk.COMPLETED.value:
status = Status.COMPLETED.value
elif status == StatusSplunk.ERROR.value:
status = Status.ERROR.value
elif content['isFinalized'] is True:
status = Status.CANCELED.value
else:
status = Status.RUNNING.value
# Construct a response object
return_obj = dict()
if response_code == 200:
return_obj['success'] = True
return_obj['status'] = status
return_obj['progress'] = progress
else:
ErrorResponder.fill_error(return_obj, response_dict, ['messages',0,'text'])
return return_obj
def create_status_connection(self, search_id):
"""
Fetching the progress and the status of the search id
:param search_id: str, search id
:return: dict
"""
return_obj = dict()
response_dict = dict()
limit = DEFAULT_LIMIT
try:
if ':' in search_id:
search_id, limit = search_id.split(':')
query = dict()
query['queryId'] = search_id
response_dict = self.client.get_query_results(**query)
return_obj['success'] = True
return_obj['status'] = self._getstatus(response_dict['status'])
results = len(response_dict['results'])
if return_obj['status'] == 'COMPLETED':
return_obj['progress'] = 100
elif return_obj['status'] == 'RUNNING':
progress = (results / int(limit)) * 100
progress_floor = math.floor(progress)
return_obj['progress'] = progress_floor
if return_obj['progress'] >= 100:
return_obj['status'] = 'COMPLETED'
else:
return_obj['progress'] = 0
except Exception as ex:
response_dict['__type'] = ex.__class__.__name__
response_dict['message'] = ex
ErrorResponder.fill_error(return_obj, response_dict, ['message'])
return return_obj
def create_results_connection(self, search_id, offset, length):
"""
Fetching the results using search id, offset and length
:param search_id: str, search id generated in transmit query
:param offset: str, offset value
:param length: str, length value
:return: dict
"""
return_obj = dict()
response_dict = dict()
try:
query = dict()
offset = int(offset)
length = int(length)
if ':' in search_id:
search_id = search_id.split(':')[0]
total_records = offset+length
query['queryId'] = search_id
response_dict = self.client.get_query_results(**query)
return_obj['success'] = True
results = response_dict['results'][offset:total_records]
result_list = []
self.format_results(result_list, results, return_obj)
except Exception as ex:
response_dict['__type'] = ex.__class__.__name__
response_dict['message'] = ex
ErrorResponder.fill_error(return_obj, response_dict, ['message'])
self.logger.debug('Return Object: {}'.format(json.dumps(return_obj, indent=4)))
return return_obj
def create_results_connection(self, search_id, offset, length):
try:
min_range = offset
max_range = int(offset) + int(length)
# Grab the response, extract the response code, and convert it to readable json
response = self.api_client.get_search_results(
search_id, min_range, max_range)
response_code = response.code
# Construct a response object
return_obj = dict()
if response_code == 200:
return_obj['success'] = True
if hasattr(response, 'content'):
data = json.loads(response.content)
else:
data = json.loads(response.read())
#print("+++++++++++++++++data ="+json.dumps(data))
if type(data) == dict and 'ID' in data.keys() and 'Message' in data.keys() and data['ID'] == 0 and\
'The Query did not retrieve any records' == data['Message']:
data = []
return_obj['data'] = data
else:
ErrorResponder.fill_error(return_obj,
response, ['message'],
connector=self.connector)
return return_obj
except Exception as err:
self.logger.error(
'error when getting search results: {}'.format(err))
raise
def generate_token(self, connection, configuration):
"""To generate the Token
:param connection: dict, connection dict
:param configuration: dict,config dict"""
return_obj = dict()
authority_url = ('https://login.windows.net/' +
configuration['auth']['tenant'])
resource = "https://" + str(connection.get('host'))
try:
context = adal.AuthenticationContext(
authority_url,
validate_authority=configuration['auth']['tenant'] != 'adfs',
)
response_dict = context.acquire_token_with_client_credentials(
resource, configuration['auth']['clientId'],
configuration['auth']['clientSecret'])
return_obj['success'] = True
return_obj['access_token'] = response_dict['accessToken']
except Exception as ex:
if ex.__class__.__name__ == 'AdalError':
response_dict = ex.error_response
ErrorResponder.fill_error(return_obj,
response_dict, ['error_description'],
connector=self.connector)
else:
ErrorResponder.fill_error(return_obj,
message=str(ex),
connector=self.connector)
Connector.logger.error("Token generation Failed: " +
str(ex.error_response))
return return_obj
def create_status_connection(self, search_id):
"""
Return status dictionary object for given query ID
:param search_id: int
:return: dict
"""
response_txt = None
return_obj = dict()
try:
response = self.api_client.get_sync_query_results(self.RELEVANCE)
response_txt = response.read().decode('utf-8')
client_count = self.DEFAULT_CLIENT_COUNT
search = re.search(self.PATTERN, response_txt, re.IGNORECASE)
if search:
client_count = search.group(1)
client_count = int(client_count)
return_obj = self.status_api_response(search_id, client_count)
except Exception as e:
if e.__class__.__name__ in ['ConnectionError', 'ProxyError']:
ErrorResponder.fill_error(return_obj, message='API call disconnected/interrupted', connector=self.connector)
return_obj['status'] = Status.ERROR.value
elif response_txt is not None:
ErrorResponder.fill_error(return_obj, message='unexpected exception', connector=self.connector)
self.logger.error('can not parse response: ' + str(response_txt))
else:
raise e
return return_obj
def create_results_connection(self, search_id, offset, length):
try:
min_range = int(offset)
max_range = int(offset) + int(length)
# Grab the response, extract the response code, and convert it to readable json
response_dict = self.api_client.get_search_results(
search_id, min_range, max_range)
response_code = response_dict["code"]
# Construct a response object
return_obj = dict()
if response_code == 200:
return_obj['success'] = True
return_obj['data'] = response_dict['data']
else:
ErrorResponder.fill_error(return_obj, response_dict,
['message'])
return return_obj
except Exception as err:
self.logger.error('error when getting search results: %s',
err,
exc_info=True)
raise
def create_status_connection(self, search_id):
try:
response_dict = self.api_client.get_search_status(search_id)
# Based on the response
# return_obj['success'] = True or False
# return_obj['status'] = One of the statuses as defined in the Status class:
# Status.RUNNING, Status.COMPLETED, Status.CANCELED, Status.ERROR
# return_obj['progress'] = Some progress code if returned from the API
# Construct a response object
response_code = response_dict["code"]
return_obj = dict()
if response_code == 200:
return_obj['success'] = True
return_obj['status'] = self.__getStatus(
response_dict["status"])
else:
return_obj['success'] = False
ErrorResponder.fill_error(return_obj,
response_dict, ['message'],
connector=self.connector)
return return_obj
except Exception as err:
self.logger.error(
'error when getting search status: {}'.format(err))
raise
def create_status_connection(self, search_id):
"""
Fetching the progress and the status of the search id
:param search_id: str, search id
:return: dict
"""
return_obj = dict()
limit = DEFAULT_LIMIT
user_limit = None
try:
search_id_length = len(search_id.split(':'))
search_id_values = search_id.split(':')
if search_id_length == 2:
search_session_id, user_session_id = search_id_values
elif search_id_length == 3:
search_session_id, user_session_id, user_limit = search_id_values
else:
raise SyntaxError("Invalid search_id format : " + str(search_id))
if user_limit and int(user_limit) <= DEFAULT_LIMIT:
limit = user_limit
response = self.api_client.get_search_status(search_session_id, user_session_id)
raw_response = response.read()
response_code = response.code
self.status_progress(return_obj, response_code, raw_response, limit)
except Exception as err:
response_error = err
ErrorResponder.fill_error(return_obj, response_error, ['message'], connector=self.connector)
return return_obj
def status_progress(self, return_obj, response_code, raw_response, limit):
"""
status progress calculation
:param return_obj: dict, building return response dict
:param raw_response: str, Api response,
:param response_code: int, Api call response code
:param limit: int, limit for status calculation """
if 199 < response_code < 300:
response_dict = json.loads(raw_response)
return_obj['success'] = True
return_obj['status'] = self.__getStatus(response_dict['status'])
results = int(response_dict['hit'])
if return_obj['status'] == 'COMPLETED':
return_obj['progress'] = 100
elif return_obj['status'] == 'RUNNING':
progress = (results / int(limit)) * 100
progress_floor = math.floor(progress)
return_obj['progress'] = progress_floor
if return_obj['progress'] >= 100:
return_obj['progress'] = 100
return_obj['status'] = 'COMPLETED'
else:
return_obj['progress'] = 0
# arcsight logger error codes - currently unavailable state
elif response_code in [500, 503]:
response_string = raw_response.decode()
ErrorResponder.fill_error(return_obj, response_string, ['message'], connector=self.connector)
elif isinstance(json.loads(raw_response), dict):
response_error = json.loads(raw_response)
response_dict = response_error['errors'][0]
ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector)
else:
raise Exception(raw_response)
def ping_connection(self):
response_txt = None
return_obj = {}
try:
response = self.api_client.ping_box()
response_code = response.code
response_txt = response.read().decode('utf-8')
if 199 < response_code < 300:
return_obj['success'] = True
elif isinstance(json.loads(response_txt), dict):
response_error_ping = json.loads(response_txt)
response_dict = response_error_ping['errors'][0]
ErrorResponder.fill_error(return_obj,
response_dict, ['message'],
connector=self.connector)
else:
raise Exception(response_txt)
except Exception as e:
if response_txt is not None:
ErrorResponder.fill_error(return_obj,
message='unexpected exception',
connector=self.connector)
self.logger.error('can not parse response: ' +
str(response_txt))
else:
raise e
return return_obj
def create_results_connection(self, query, offset, length):
response_txt = None
return_obj = dict()
try:
response = self.api_client.run_search(query, offset, length)
return_obj = self._handle_errors(response, return_obj)
if (return_obj['success']):
response_json = json.loads(return_obj["data"])
if response_json['hits']:
# and (response_json['hits']['total']['value'] >= 0 or response_json['hits']['total'] >= 0):
self.logger.error("Total # of hits:" +
str(response_json['hits']['total']))
return_obj['data'] = [
record['_source']
for record in response_json["hits"]["hits"]
]
self.logger.error("Total # of records: " +
str(len(return_obj['data'])))
return return_obj
except Exception as e:
if response_txt is not None:
ErrorResponder.fill_error(return_obj,
message='unexpected exception')
self.logger.error('can not parse response: ' +
str(response_txt))
else:
raise e
def delete_query_connection(self, search_id):
response = self.api_client.delete_search(search_id)
response_code = response.code
response_text = response.read()
error = None
response_dict = dict()
try:
# Successful deletion returns a 204 status code and empty response
if response_text:
response_dict = json.loads(response_text)
except ValueError as ex:
self.logger.debug(response_text)
error = Exception(f'Can not parse response: {ex}')
# Construct a response object
return_obj = dict()
return_obj['success'] = False
# Successful deletion returns a 204 status code and empty response
if response_code == 204:
return_obj['success'] = True
else:
ErrorResponder.fill_error(return_obj,
response_dict, ['message'],
error=error,
connector=self.connector)
return return_obj
def create_query_connection(self, query):
# Grab the response, extract the response code, and convert it to readable json
response = self.api_client.create_search(query)
response_code = response.code
response_text = response.read()
error = None
response_dict = dict()
try:
response_dict = json.loads(response_text)
except ValueError as ex:
self.logger.debug(response_text)
error = Exception(
f'Can not parse response: {ex} : {response_text}')
# Construct a response object
return_obj = dict()
if response_code == 201:
return_obj['success'] = True
return_obj['search_id'] = response_dict['search_id']
else:
ErrorResponder.fill_error(return_obj,
response_dict, ['message'],
error=error)
return return_obj
def create_status_connection(self, search_id):
# Grab the response, extract the response code, and convert it to readable json
response = self.api_client.get_search(search_id)
response_code = response.code
response_text = response.read()
error = None
response_dict = dict()
try:
response_dict = json.loads(response_text)
except Exception as ex:
self.logger.debug(response_text)
error = Exception(
f'Can not parse response: {ex} : {response_text}')
# Construct a response object
return_obj = dict()
if response_code == 200:
return_obj['success'] = True
return_obj['status'] = self.__getStatus(response_dict['status'])
return_obj['progress'] = response_dict['progress']
else:
ErrorResponder.fill_error(return_obj,
response_dict, ['message'],
error=error,
connector=self.connector)
return return_obj
def create_results_connection(self, search_id, offset, length):
min_range = offset
max_range = offset + length
# Grab the response, extract the response code, and convert it to readable json
response = self.api_client.get_search_results(search_id, 'application/json', min_range, max_range)
response_code = response.code
# Construct a response object
return_obj = dict()
error = None
response_text = response.read()
response_dict = dict()
try:
response_dict = json.loads(response_text)
except ValueError as ex:
self.logger.debug(response_text)
error = Exception(f'Can not parse response from Qradar server. The response is not a valid json: {response_text} : {ex}')
if 200 <= response_code <= 299 and error is None:
return_obj['success'] = True
return_obj['data'] = response_dict.get('events', response_dict.get('flows'))
else:
ErrorResponder.fill_error(return_obj, response_dict, ['message'], error=error, connector=self.connector)
return return_obj
def generate_token(connection, configuration):
"""To generate the Token
:param connection: dict, connection dict
:param configuration: dict,config dict"""
authority_url = ('https://login.windows.net/' +
configuration['auth']['tenant'])
resource = "https://" + str(connection.get('host'))
try:
context = adal.AuthenticationContext(
authority_url,
validate_authority=configuration['auth']['tenant'] != 'adfs',
)
token = context.acquire_token_with_client_credentials(
resource, configuration['auth']['clientId'],
configuration['auth']['clientSecret'])
token_value = token['accessToken']
return token_value
except Exception as ex:
return_obj = dict()
if ex.error_response:
ErrorResponder.fill_error(return_obj, ex.error_response,
['reason'])
print("Token generation Failed:", return_obj)
raise ex
def create_query_connection(self, query):
response = self.api_client.create_search(query)
response_code = response.code
response_text = response.read()
error = None
response_dict = dict()
try:
response_dict = json.loads(response_text)
except ValueError as ex:
self.logger.debug(response_text)
error = Exception(f'Can not parse response: {ex}')
return_obj = dict()
return_obj['success'] = False
if response_dict and response_code == 200:
return_obj['success'] = True
return_obj['search_id'] = response_dict['job_id']
else:
ErrorResponder.fill_error(return_obj,
response_dict, ['message'],
error=error)
return return_obj
def translate_results(self, data_source, data):
translator = self.get_results_translator()
try:
return translator.translate_results(data_source, data)
except Exception as ex:
result = {}
ErrorResponder.fill_error(result, message_struct={'exception': ex})
return result