def api_request(self, ro): """ """ api_response = None fail_msg = None h_content_length = None h_content_type = None start = datetime.now() # # enable activity log # if self._activity_log: ro.enable_activity_log() # # prepare request # url = '{0!s}{1!s}'.format(self._api_url, ro.request_uri) api_request = Request(ro.http_method, url, data=ro.body, params=ro.payload) request_prepped = api_request.prepare() # # generate headers # ro.set_path_url(request_prepped.path_url) self._api_request_headers(ro) request_prepped.prepare_headers(ro.headers) # # Debug # self.tcl.debug('request_object: {0!s}'.format(ro)) self.tcl.debug('url: {0!s}'.format(url)) self.tcl.debug('path url: {0!s}'.format(request_prepped.path_url)) # # api request (gracefully handle temporary communications issues with the API) # for i in range(1, self._api_retries + 1, 1): try: api_response = self._session.send( request_prepped, verify=self._verify_ssl, timeout=self._api_request_timeout, proxies=self._proxies, stream=False) break except exceptions.ReadTimeout as e: self.tcl.error('Error: {0!s}'.format(e)) self.tcl.error('The server may be experiencing delays at the moment.') self.tcl.info('Pausing for {0!s} seconds to give server time to catch up.'.format(self._api_sleep)) time.sleep(self._api_sleep) self.tcl.info('Retry {0!s} ....'.format(i)) if i == self._api_retries: self.tcl.critical('Exiting: {0!s}'.format(e)) raise RuntimeError(e) except exceptions.ConnectionError as e: self.tcl.error('Error: {0!s}'.format(e)) self.tcl.error('Connection Error. The server may be down.') self.tcl.info('Pausing for {0!s} seconds to give server time to catch up.'.format(self._api_sleep)) time.sleep(self._api_sleep) self.tcl.info('Retry {0!s} ....'.format(i)) if i == self._api_retries: self.tcl.critical('Exiting: {0!s}'.format(e)) raise RuntimeError(e) except socket.error as e: self.tcl.critical('Exiting: {0!s}'.format(e)) raise RuntimeError(e) # # header values # if 'content-length' in api_response.headers: h_content_length = api_response.headers['content-length'] if 'content-type' in api_response.headers: h_content_type = api_response.headers['content-type'] # # raise exception on *critical* errors # non_critical_errors = [ b'The MD5 for this File is invalid, a File with this MD5 already exists', # 400 (application/json) b'The SHA-1 for this File is invalid, a File with this SHA-1 already exists', # 400 (application/json) b'The SHA-256 for this File is invalid, a File with this SHA-256 already exists', # 400 (application/json) b'The requested resource was not found', # 404 (application/json) b'Could not find resource for relative', # 500 (text/plain) b'The requested Security Label was not removed - access was denied', # 401 (application/json) ] # # TODO: work out some logic to improve the API error handling, possible area where API could improve # # valid status codes 200, 201, 202 # if api_response.status_code in [400, 401, 403, 500, 503]: if api_response.status_code not in [200, 201, 202]: # check for non critical errors that have bad status codes nce_found = False fail_msg = api_response.content for nce in non_critical_errors: # api_response_dict['message'] not in non_critical_errors: if re.findall(nce, api_response.content): nce_found = True break if ro.failure_callback is not None: ro.failure_callback(api_response.status_code) # raise error on bad status codes that are not defined as nce if not nce_found: self.tcl.critical('Status Code: {0:d}'.format(api_response.status_code)) self.tcl.critical('Failed API Response: {0!s}'.format(api_response.content)) if ro.failure_callback is not None: ro.failure_callback(api_response.status_code) raise RuntimeError(api_response.content) # # set response encoding (best guess) # if api_response.encoding is None: api_response.encoding = api_response.apparent_encoding # # Debug # self.tcl.debug('url: %s', api_response.url) self.tcl.debug('status_code: %s', api_response.status_code) self.tcl.debug('content-length: %s', h_content_length) self.tcl.debug('content-type: %s', h_content_type) # # Report # self.report.add_api_call() # count api calls self.report.add_request_time(datetime.now() - start) self.tcl.debug('Request Time: {0!s}'.format(datetime.now() - start)) if self._enable_report: report_entry = ReportEntry() report_entry.add_request_object(ro) report_entry.set_request_url(api_response.url) report_entry.set_status_code(api_response.status_code) report_entry.set_failure_msg(fail_msg) self.report.add(report_entry) # # return response # # self.print_mem('end _api_request') return api_response
def api_filter_handler(self, resource_obj, filter_objs): """ """ data_set = None if not filter_objs: # build api call (no filters) default_request_object = resource_obj.default_request_object data_set = self.api_response_handler(resource_obj, default_request_object) else: # # process each filter added to the resource object for retrieve # first_run = True # # each resource object can have x filter objects with an operator to join or intersect results # for filter_obj in filter_objs: obj_list = [] # temp storage for results on individual filter objects owners = filter_obj.owners if len(owners) == 0: # handle filters with no owners owners = [self._api_org] # use default org # iterate through all owners for o in owners: self.tcl.debug('owner: {0!s}'.format(o)) if len(filter_obj) > 0: # request object are for api filters for ro in filter_obj: if ro.owner_allowed: ro.set_owner(o) results = self.api_response_handler(resource_obj, ro) if ro.resource_type not in [ResourceType.OWNERS, ResourceType.VICTIMS, ResourceType.BATCH_JOBS]: # TODO: should this be done? # post filter owners for obj in results: if obj.owner_name.upper() != o.upper(): results.remove(obj) obj_list.extend(results) else: ro = filter_obj.default_request_object if ro.owner_allowed: ro.set_owner(o) results = self.api_response_handler(resource_obj, ro) if ro.resource_type not in [ResourceType.OWNERS, ResourceType.VICTIMS]: # TODO: should this be done? # post filter owners for obj in results: if obj.owner_name.upper() != o.upper(): results.remove(obj) obj_list.extend(results) # # post filters # pf_obj_set = set(obj_list) self.tcl.debug('count before post filter: {0:d}'.format(len(obj_list))) for pfo in filter_obj.post_filters: self.tcl.debug('pfo: {0!s}'.format(pfo)) # # Report Entry # report_entry = ReportEntry() report_entry.add_post_filter_object(pfo) # current post filter method filter_method = getattr(resource_obj, pfo.method) # current post filter results post_filter_results = set(filter_method(pfo.filter, pfo.operator, pfo.description)) pf_obj_set = pf_obj_set.intersection(post_filter_results) self.report.add(report_entry) # set obj_list to post_filter results if filter_obj.post_filters_len > 0: obj_list = list(pf_obj_set) self.tcl.debug('count after post filter: {0:d}'.format(len(obj_list))) # no need to join or intersect on first run if first_run: data_set = set(obj_list) first_run = False continue # # depending on the filter type the result will be intersected or joined # if filter_obj.operator is FilterSetOperator.AND: data_set = data_set.intersection(obj_list) elif filter_obj.operator is FilterSetOperator.OR: data_set.update(set(obj_list)) # # only add to report if these results should be tracked (exclude attribute, tags, etc) # self.report.add_filtered_results(len(data_set)) # # after intersection or join add the objects to the resource object # for obj in data_set: resource_obj.add_obj(obj)