def api_request(self, ro):
""" """
api_response = None
fail_msg = None
h_content_length = None
h_content_type = None
start = datetime.now()
#
# enable activity log
#
if self._activity_log:
ro.enable_activity_log()
#
# prepare request
#
url = '{0!s}{1!s}'.format(self._api_url, ro.request_uri)
api_request = Request(ro.http_method, url, data=ro.body, params=ro.payload)
request_prepped = api_request.prepare()
#
# generate headers
#
ro.set_path_url(request_prepped.path_url)
self._api_request_headers(ro)
request_prepped.prepare_headers(ro.headers)
#
# Debug
#
self.tcl.debug('request_object: {0!s}'.format(ro))
self.tcl.debug('url: {0!s}'.format(url))
self.tcl.debug('path url: {0!s}'.format(request_prepped.path_url))
#
# api request (gracefully handle temporary communications issues with the API)
#
for i in range(1, self._api_retries + 1, 1):
try:
api_response = self._session.send(
request_prepped, verify=self._verify_ssl, timeout=self._api_request_timeout,
proxies=self._proxies, stream=False)
break
except exceptions.ReadTimeout as e:
self.tcl.error('Error: {0!s}'.format(e))
self.tcl.error('The server may be experiencing delays at the moment.')
self.tcl.info('Pausing for {0!s} seconds to give server time to catch up.'.format(self._api_sleep))
time.sleep(self._api_sleep)
self.tcl.info('Retry {0!s} ....'.format(i))
if i == self._api_retries:
self.tcl.critical('Exiting: {0!s}'.format(e))
raise RuntimeError(e)
except exceptions.ConnectionError as e:
self.tcl.error('Error: {0!s}'.format(e))
self.tcl.error('Connection Error. The server may be down.')
self.tcl.info('Pausing for {0!s} seconds to give server time to catch up.'.format(self._api_sleep))
time.sleep(self._api_sleep)
self.tcl.info('Retry {0!s} ....'.format(i))
if i == self._api_retries:
self.tcl.critical('Exiting: {0!s}'.format(e))
raise RuntimeError(e)
except socket.error as e:
self.tcl.critical('Exiting: {0!s}'.format(e))
raise RuntimeError(e)
#
# header values
#
if 'content-length' in api_response.headers:
h_content_length = api_response.headers['content-length']
if 'content-type' in api_response.headers:
h_content_type = api_response.headers['content-type']
#
# raise exception on *critical* errors
#
non_critical_errors = [
b'The MD5 for this File is invalid, a File with this MD5 already exists', # 400 (application/json)
b'The SHA-1 for this File is invalid, a File with this SHA-1 already exists', # 400 (application/json)
b'The SHA-256 for this File is invalid, a File with this SHA-256 already exists', # 400 (application/json)
b'The requested resource was not found', # 404 (application/json)
b'Could not find resource for relative', # 500 (text/plain)
b'The requested Security Label was not removed - access was denied', # 401 (application/json)
]
#
# TODO: work out some logic to improve the API error handling, possible area where API could improve
#
# valid status codes 200, 201, 202
# if api_response.status_code in [400, 401, 403, 500, 503]:
if api_response.status_code not in [200, 201, 202]:
# check for non critical errors that have bad status codes
nce_found = False
fail_msg = api_response.content
for nce in non_critical_errors:
# api_response_dict['message'] not in non_critical_errors:
if re.findall(nce, api_response.content):
nce_found = True
break
if ro.failure_callback is not None:
ro.failure_callback(api_response.status_code)
# raise error on bad status codes that are not defined as nce
if not nce_found:
self.tcl.critical('Status Code: {0:d}'.format(api_response.status_code))
self.tcl.critical('Failed API Response: {0!s}'.format(api_response.content))
if ro.failure_callback is not None:
ro.failure_callback(api_response.status_code)
raise RuntimeError(api_response.content)
#
# set response encoding (best guess)
#
if api_response.encoding is None:
api_response.encoding = api_response.apparent_encoding
#
# Debug
#
self.tcl.debug('url: %s', api_response.url)
self.tcl.debug('status_code: %s', api_response.status_code)
self.tcl.debug('content-length: %s', h_content_length)
self.tcl.debug('content-type: %s', h_content_type)
#
# Report
#
self.report.add_api_call() # count api calls
self.report.add_request_time(datetime.now() - start)
self.tcl.debug('Request Time: {0!s}'.format(datetime.now() - start))
if self._enable_report:
report_entry = ReportEntry()
report_entry.add_request_object(ro)
report_entry.set_request_url(api_response.url)
report_entry.set_status_code(api_response.status_code)
report_entry.set_failure_msg(fail_msg)
self.report.add(report_entry)
#
# return response
#
# self.print_mem('end _api_request')
return api_response
def api_filter_handler(self, resource_obj, filter_objs):
""" """
data_set = None
if not filter_objs:
# build api call (no filters)
default_request_object = resource_obj.default_request_object
data_set = self.api_response_handler(resource_obj, default_request_object)
else:
#
# process each filter added to the resource object for retrieve
#
first_run = True
#
# each resource object can have x filter objects with an operator to join or intersect results
#
for filter_obj in filter_objs:
obj_list = [] # temp storage for results on individual filter objects
owners = filter_obj.owners
if len(owners) == 0: # handle filters with no owners
owners = [self._api_org] # use default org
# iterate through all owners
for o in owners:
self.tcl.debug('owner: {0!s}'.format(o))
if len(filter_obj) > 0:
# request object are for api filters
for ro in filter_obj:
if ro.owner_allowed:
ro.set_owner(o)
results = self.api_response_handler(resource_obj, ro)
if ro.resource_type not in [ResourceType.OWNERS,
ResourceType.VICTIMS,
ResourceType.BATCH_JOBS]:
# TODO: should this be done?
# post filter owners
for obj in results:
if obj.owner_name.upper() != o.upper():
results.remove(obj)
obj_list.extend(results)
else:
ro = filter_obj.default_request_object
if ro.owner_allowed:
ro.set_owner(o)
results = self.api_response_handler(resource_obj, ro)
if ro.resource_type not in [ResourceType.OWNERS, ResourceType.VICTIMS]:
# TODO: should this be done?
# post filter owners
for obj in results:
if obj.owner_name.upper() != o.upper():
results.remove(obj)
obj_list.extend(results)
#
# post filters
#
pf_obj_set = set(obj_list)
self.tcl.debug('count before post filter: {0:d}'.format(len(obj_list)))
for pfo in filter_obj.post_filters:
self.tcl.debug('pfo: {0!s}'.format(pfo))
#
# Report Entry
#
report_entry = ReportEntry()
report_entry.add_post_filter_object(pfo)
# current post filter method
filter_method = getattr(resource_obj, pfo.method)
# current post filter results
post_filter_results = set(filter_method(pfo.filter, pfo.operator, pfo.description))
pf_obj_set = pf_obj_set.intersection(post_filter_results)
self.report.add(report_entry)
# set obj_list to post_filter results
if filter_obj.post_filters_len > 0:
obj_list = list(pf_obj_set)
self.tcl.debug('count after post filter: {0:d}'.format(len(obj_list)))
# no need to join or intersect on first run
if first_run:
data_set = set(obj_list)
first_run = False
continue
#
# depending on the filter type the result will be intersected or joined
#
if filter_obj.operator is FilterSetOperator.AND:
data_set = data_set.intersection(obj_list)
elif filter_obj.operator is FilterSetOperator.OR:
data_set.update(set(obj_list))
#
# only add to report if these results should be tracked (exclude attribute, tags, etc)
#
self.report.add_filtered_results(len(data_set))
#
# after intersection or join add the objects to the resource object
#
for obj in data_set:
resource_obj.add_obj(obj)
