def api_request(self, ro): """ """ api_response = None fail_msg = None h_content_length = None h_content_type = None start = datetime.now() # # enable activity log # if self._activity_log: ro.enable_activity_log() # # prepare request # url = '{0!s}{1!s}'.format(self._api_url, ro.request_uri) api_request = Request(ro.http_method, url, data=ro.body, params=ro.payload) request_prepped = api_request.prepare() # # generate headers # ro.set_path_url(request_prepped.path_url) self._api_request_headers(ro) request_prepped.prepare_headers(ro.headers) # # Debug # self.tcl.debug('request_object: {0!s}'.format(ro)) self.tcl.debug('url: {0!s}'.format(url)) self.tcl.debug('path url: {0!s}'.format(request_prepped.path_url)) # # api request (gracefully handle temporary communications issues with the API) # for i in range(1, self._api_retries + 1, 1): try: api_response = self._session.send( request_prepped, verify=self._verify_ssl, timeout=self._api_request_timeout, proxies=self._proxies, stream=False) break except exceptions.ReadTimeout as e: self.tcl.error('Error: {0!s}'.format(e)) self.tcl.error('The server may be experiencing delays at the moment.') self.tcl.info('Pausing for {0!s} seconds to give server time to catch up.'.format(self._api_sleep)) time.sleep(self._api_sleep) self.tcl.info('Retry {0!s} ....'.format(i)) if i == self._api_retries: self.tcl.critical('Exiting: {0!s}'.format(e)) raise RuntimeError(e) except exceptions.ConnectionError as e: self.tcl.error('Error: {0!s}'.format(e)) self.tcl.error('Connection Error. The server may be down.') self.tcl.info('Pausing for {0!s} seconds to give server time to catch up.'.format(self._api_sleep)) time.sleep(self._api_sleep) self.tcl.info('Retry {0!s} ....'.format(i)) if i == self._api_retries: self.tcl.critical('Exiting: {0!s}'.format(e)) raise RuntimeError(e) except socket.error as e: self.tcl.critical('Exiting: {0!s}'.format(e)) raise RuntimeError(e) # # header values # if 'content-length' in api_response.headers: h_content_length = api_response.headers['content-length'] if 'content-type' in api_response.headers: h_content_type = api_response.headers['content-type'] # # raise exception on *critical* errors # non_critical_errors = [ b'The MD5 for this File is invalid, a File with this MD5 already exists', # 400 (application/json) b'The SHA-1 for this File is invalid, a File with this SHA-1 already exists', # 400 (application/json) b'The SHA-256 for this File is invalid, a File with this SHA-256 already exists', # 400 (application/json) b'The requested resource was not found', # 404 (application/json) b'Could not find resource for relative', # 500 (text/plain) b'The requested Security Label was not removed - access was denied', # 401 (application/json) ] # # TODO: work out some logic to improve the API error handling, possible area where API could improve # # valid status codes 200, 201, 202 # if api_response.status_code in [400, 401, 403, 500, 503]: if api_response.status_code not in [200, 201, 202]: # check for non critical errors that have bad status codes nce_found = False fail_msg = api_response.content for nce in non_critical_errors: # api_response_dict['message'] not in non_critical_errors: if re.findall(nce, api_response.content): nce_found = True break if ro.failure_callback is not None: ro.failure_callback(api_response.status_code) # raise error on bad status codes that are not defined as nce if not nce_found: self.tcl.critical('Status Code: {0:d}'.format(api_response.status_code)) self.tcl.critical('Failed API Response: {0!s}'.format(api_response.content)) if ro.failure_callback is not None: ro.failure_callback(api_response.status_code) raise RuntimeError(api_response.content) # # set response encoding (best guess) # if api_response.encoding is None: api_response.encoding = api_response.apparent_encoding # # Debug # self.tcl.debug('url: %s', api_response.url) self.tcl.debug('status_code: %s', api_response.status_code) self.tcl.debug('content-length: %s', h_content_length) self.tcl.debug('content-type: %s', h_content_type) # # Report # self.report.add_api_call() # count api calls self.report.add_request_time(datetime.now() - start) self.tcl.debug('Request Time: {0!s}'.format(datetime.now() - start)) if self._enable_report: report_entry = ReportEntry() report_entry.add_request_object(ro) report_entry.set_request_url(api_response.url) report_entry.set_status_code(api_response.status_code) report_entry.set_failure_msg(fail_msg) self.report.add(report_entry) # # return response # # self.print_mem('end _api_request') return api_response
def api_filter_handler(self, resource_obj, filter_objs): """ """ data_set = None if not filter_objs: # build api call (no filters) default_request_object = resource_obj.default_request_object data_set = self.api_response_handler(resource_obj, default_request_object) else: # # process each filter added to the resource object for retrieve # first_run = True # # each resource object can have x filter objects with an operator to join or intersect results # for filter_obj in filter_objs: obj_list = [] # temp storage for results on individual filter objects owners = filter_obj.owners if len(owners) == 0: # handle filters with no owners owners = [self._api_org] # use default org # iterate through all owners for o in owners: self.tcl.debug('owner: {0!s}'.format(o)) if len(filter_obj) > 0: # request object are for api filters for ro in filter_obj: if ro.owner_allowed: ro.set_owner(o) results = self.api_response_handler(resource_obj, ro) if ro.resource_type not in [ResourceType.OWNERS, ResourceType.VICTIMS, ResourceType.BATCH_JOBS]: # TODO: should this be done? # post filter owners for obj in results: if obj.owner_name.upper() != o.upper(): results.remove(obj) obj_list.extend(results) else: ro = filter_obj.default_request_object if ro.owner_allowed: ro.set_owner(o) results = self.api_response_handler(resource_obj, ro) if ro.resource_type not in [ResourceType.OWNERS, ResourceType.VICTIMS]: # TODO: should this be done? # post filter owners for obj in results: if obj.owner_name.upper() != o.upper(): results.remove(obj) obj_list.extend(results) # # post filters # pf_obj_set = set(obj_list) self.tcl.debug('count before post filter: {0:d}'.format(len(obj_list))) for pfo in filter_obj.post_filters: self.tcl.debug('pfo: {0!s}'.format(pfo)) # # Report Entry # report_entry = ReportEntry() report_entry.add_post_filter_object(pfo) # current post filter method filter_method = getattr(resource_obj, pfo.method) # current post filter results post_filter_results = set(filter_method(pfo.filter, pfo.operator, pfo.description)) pf_obj_set = pf_obj_set.intersection(post_filter_results) self.report.add(report_entry) # set obj_list to post_filter results if filter_obj.post_filters_len > 0: obj_list = list(pf_obj_set) self.tcl.debug('count after post filter: {0:d}'.format(len(obj_list))) # no need to join or intersect on first run if first_run: data_set = set(obj_list) first_run = False continue # # depending on the filter type the result will be intersected or joined # if filter_obj.operator is FilterSetOperator.AND: data_set = data_set.intersection(obj_list) elif filter_obj.operator is FilterSetOperator.OR: data_set.update(set(obj_list)) # # only add to report if these results should be tracked (exclude attribute, tags, etc) # self.report.add_filtered_results(len(data_set)) # # after intersection or join add the objects to the resource object # for obj in data_set: resource_obj.add_obj(obj)
def api_request(self, ro): """ """ api_response = None fail_msg = None h_content_length = None h_content_type = None start = datetime.now() # # enable activity log # # request_object.enable_activity_mode() # # prepare request # url = '{0!s}{1!s}'.format(self._api_url, ro.request_uri) api_request = Request(ro.http_method, url, data=ro.body, params=ro.payload) request_prepped = api_request.prepare() # # generate headers # ro.set_path_url(request_prepped.path_url) self._api_request_headers(ro) request_prepped.prepare_headers(ro.headers) # # Debug # self.tcl.debug('request_object: {0!s}'.format(ro)) self.tcl.debug('url: {0!s}'.format(url)) self.tcl.debug('path url: {0!s}'.format(request_prepped.path_url)) # # api request (gracefully handle temporary communications issues with the API) # for i in range(1, self._api_retries + 1, 1): try: api_response = self._session.send( request_prepped, verify=self._verify_ssl, timeout=self._api_request_timeout, proxies=self._proxies, stream=False) break except exceptions.ReadTimeout as e: self.tcl.error('Error: {0!s}'.format(e)) self.tcl.error('The server may be experiencing delays at the moment.') self.tcl.info('Pausing for {0!s} seconds to give server time to catch up.'.format(self._api_sleep)) time.sleep(self._api_sleep) self.tcl.info('Retry {0!s} ....'.format(i)) if i == self._api_retries: self.tcl.critical('Exiting: {0!s}'.format(e)) raise RuntimeError(e) except exceptions.ConnectionError as e: self.tcl.error('Error: {0!s}'.format(e)) self.tcl.error('Connection Error. The server may be down.') self.tcl.info('Pausing for {0!s} seconds to give server time to catch up.'.format(self._api_sleep)) time.sleep(self._api_sleep) self.tcl.info('Retry {0!s} ....'.format(i)) if i == self._api_retries: self.tcl.critical('Exiting: {0!s}'.format(e)) raise RuntimeError(e) except socket.error as e: self.tcl.critical('Exiting: {0!s}'.format(e)) raise RuntimeError(e) # # header values # if 'content-length' in api_response.headers: h_content_length = api_response.headers['content-length'] if 'content-type' in api_response.headers: h_content_type = api_response.headers['content-type'] # # raise exception on *critical* errors # non_critical_errors = [ b'The MD5 for this File is invalid, a File with this MD5 already exists', # 400 (application/json) b'The SHA-1 for this File is invalid, a File with this SHA-1 already exists', # 400 (application/json) b'The SHA-256 for this File is invalid, a File with this SHA-256 already exists', # 400 (application/json) b'The requested resource was not found', # 404 (application/json) b'Could not find resource for relative', # 500 (text/plain) b'The requested Security Label was not removed - access was denied', # 401 (application/json) ] # # TODO: work out some logic to improve the API error handling, possible area where API could improve # # valid status codes 200, 201, 202 # if api_response.status_code in [400, 401, 403, 500, 503]: if api_response.status_code not in [200, 201, 202]: # check for non critical errors that have bad status codes nce_found = False fail_msg = api_response.content for nce in non_critical_errors: # api_response_dict['message'] not in non_critical_errors: if re.findall(nce, api_response.content): nce_found = True break if ro.failure_callback is not None: ro.failure_callback(api_response.status_code) # raise error on bad status codes that are not defined as nce if not nce_found: self.tcl.critical('Status Code: {0:d}'.format(api_response.status_code)) self.tcl.critical('Failed API Response: {0!s}'.format(api_response.content)) if ro.failure_callback is not None: ro.failure_callback(api_response.status_code) raise RuntimeError(api_response.content) # # set response encoding (best guess) # if api_response.encoding is None: api_response.encoding = api_response.apparent_encoding # # Debug # self.tcl.debug('url: %s', api_response.url) self.tcl.debug('status_code: %s', api_response.status_code) self.tcl.debug('content-length: %s', h_content_length) self.tcl.debug('content-type: %s', h_content_type) # # Report # self.report.add_api_call() # count api calls self.report.add_request_time(datetime.now() - start) self.tcl.debug('Request Time: {0!s}'.format(datetime.now() - start)) if self._enable_report: report_entry = ReportEntry() report_entry.add_request_object(ro) report_entry.set_request_url(api_response.url) report_entry.set_status_code(api_response.status_code) report_entry.set_failure_msg(fail_msg) self.report.add(report_entry) # # return response # # self.print_mem('end _api_request') return api_response
def api_build_request(self, resource_obj, request_object, owners=None): """ """ # # initialize vars # obj_list = [] if owners is None or not owners: owners = [self._api_org] # set owners to default org else: owners = list(owners) # get copy of owners list for pop count = len(owners) modified_since = None request_payload = {} result_start = 0 result_remaining = 0 # # resource object values # body = request_object.body content_type = request_object.content_type http_method = request_object.http_method owner_allowed = request_object.owner_allowed resource_pagination = request_object.resource_pagination resource_type = request_object.resource_type request_uri = request_object.request_uri # # ReportEntry (create a report entry for this request) # report_entry = ReportEntry() report_entry.set_action(request_object.name) report_entry.set_resource_type(resource_obj.resource_type) report_entry.add_data({'HTTP Method': http_method}) report_entry.add_data({'Max Results': self._api_max_results}) report_entry.add_data({'Owners': str(owners)}) report_entry.add_data({'Owner Allowed': owner_allowed}) report_entry.add_data({'Request URI': request_uri}) report_entry.add_data({'Request Body': body}) report_entry.add_data({'Resource Pagination': resource_pagination}) report_entry.add_data({'Resource Type': resource_type}) # # debug # self.tcl.debug('Action: {0}'.format(request_object.name)) self.tcl.debug('Resource Type: {0}'.format(resource_obj.resource_type)) self.tcl.debug('HTTP Method: {0}'.format(http_method)) self.tcl.debug('Max Results: {0}'.format(self._api_max_results)) self.tcl.debug('Owners: {0}'.format(str(owners))) self.tcl.debug('Owner Allowed: {0}'.format(owner_allowed)) self.tcl.debug('Request URI: {0}'.format(request_uri)) self.tcl.debug('Request Body: {0}'.format(body)) self.tcl.debug('Resource Pagination: {0}'.format(resource_pagination)) self.tcl.debug('Resource Type: {0}'.format(resource_type)) # TODO: what would happen if this was always set to request object value? if resource_type.name in [ 'INDICATORS', 'ADDRESSES', 'EMAIL_ADDRESSES', 'FILES', 'HOSTS', 'URLS']: # TODO: find a cleaner way if not re.findall('bulk', request_uri): modified_since = resource_obj.get_modified_since() # update resource object with max results # ???moved to report resource_obj.set_max_results(self._api_max_results) # append uri to resource object # ???moved to report resource_obj.add_uris(request_uri) # iterate through all owners and results if owner_allowed or resource_pagination: # DEBUG if modified_since is not None: request_payload['modifiedSince'] = modified_since # ReportEntry report_entry.add_data({'Modified Since': modified_since}) # if request_object.owner_allowed: # # if len(list(request_object.owners)) > 0: # # owners = list(request_object.owners) # count = len(owners) for x in xrange(count): retrieve_data = True # only add_obj owner parameter if owners is allowed if owner_allowed: owner = owners.pop(0) request_payload['owner'] = owner # DEBUG self.tcl.debug('owner: %s', owner) self.tcl.debug('request_payload: %s', request_payload) # only add_obj result parameters if resource_pagination is allowed if resource_pagination: result_limit = int(self._api_max_results) result_remaining = result_limit result_start = 0 while retrieve_data: # set retrieve data to False to prevent loop for non paginating request retrieve_data = False # only add_obj result parameters if resource_pagination is allowed if request_object.resource_pagination: request_payload['resultLimit'] = result_limit request_payload['resultStart'] = result_start # DEBUG self.tcl.debug('result_limit: %s', result_limit) self.tcl.debug('result_start: %s', result_start) # # api request # api_response = self._api_request( request_uri, request_payload=request_payload, http_method=http_method, body=body) api_response.encoding = 'utf-8' # ReportEntry report_entry.set_status_code(api_response.status_code) report_entry.add_request_url(api_response.url) # break is status is not valid if api_response.status_code not in [200, 201, 202]: # ReportEntry report_entry.set_status('Failure') report_entry.set_status_code(api_response.status_code) report_entry.add_data({'Failure Message': api_response.content}) # Logging resource_obj.add_error_message(ErrorCodes.e80000.value.format(api_response.content)) break # # CSV Special Case # if re.findall('bulk/csv$', request_object.request_uri): obj_list.extend( self._api_process_response_csv(resource_obj, api_response.content)) break # # parse response # api_response_dict = api_response.json() resource_obj.current_url = api_response.url # update group object with api response data resource_obj.add_api_response(api_response.content) resource_obj.add_status_code(api_response.status_code) # resource_obj.add_error_message(api_response.content) # # bulk indicators # # indicator response has no status so it must come first if 'indicator' in api_response_dict: # # process response # obj_list.extend(self._api_process_response( resource_obj, api_response, request_object)) # # non Success status # elif api_response_dict['status'] != 'Success': # ReportEntry report_entry.set_status(api_response_dict['status']) report_entry.add_data( {'Failure Message': api_response_dict['message']}) # # normal response # elif 'data' in api_response_dict: # ReportEntry report_entry.set_status(api_response_dict['status']) # update resource object resource_obj.add_status(ApiStatus[api_response_dict['status'].upper()]) # # process response # obj_list.extend(self._api_process_response( resource_obj, api_response, request_object)) # add_obj resource_pagination if required if request_object.resource_pagination: # get the number of results returned by the api if result_start == 0: result_remaining = api_response_dict['data']['resultCount'] result_remaining -= result_limit # flip retrieve data flag if there are more results to pull if result_remaining > 0: retrieve_data = True # increment the start position result_start += result_limit else: resource_obj.add_error_message(api_response.content) elif content_type == 'application/octet-stream': # # api request # api_response = self._api_request( request_uri, request_payload={}, http_method=http_method, body=body, content_type=content_type) # ReportEntry report_entry.set_status_code(api_response.status_code) report_entry.add_request_url(api_response.url) if api_response.status_code not in [200, 201, 202]: # ReportEntry report_entry.set_status('Failure') report_entry.add_data({'Failure Message': api_response.content}) # Logging self.tcl.critical(ErrorCodes.e80000.value.format(api_response.content)) raise RuntimeError(ErrorCodes.e90001.value) else: report_entry.set_status('Success') return api_response.content else: # # api request # api_response = self._api_request( request_uri, request_payload={}, http_method=http_method, body=body) api_response.encoding = 'utf-8' if 'content-type' in api_response.headers: content_type = api_response.headers['content-type'] # ReportData report_entry.set_status_code(api_response.status_code) report_entry.add_request_url(api_response.url) # break is status is not valid if api_response.status_code not in [200, 201, 202]: if api_response.status_code == 404: # failure_message = api_response.json()['message'] failure_message = api_response.content else: failure_message = api_response.content # ReportEntry report_entry.set_status('Failure') report_entry.add_data({'Failure Message': failure_message}) # Logging self.tcl.critical(ErrorCodes.e80000.value.format(api_response.content)) raise RuntimeError(ErrorCodes.e90001.value) elif content_type == "text/plain": # signature download return api_response.content else: api_response_dict = api_response.json() resource_obj.current_url = api_response.url # ReportEntry report_entry.set_status(api_response_dict['status']) # update group object with api response data resource_obj.add_api_response(api_response.content) resource_obj.add_status_code(api_response.status_code) resource_obj.add_status(ApiStatus[api_response_dict['status'].upper()]) # no need to process data for deletes or if no data exists if http_method != 'DELETE' and 'data' in api_response_dict: # # process response # processed_data = self._api_process_response( resource_obj, api_response, request_object) obj_list.extend(processed_data) # ReportData report_entry.add_data({'Result Count': len(obj_list)}) # Report self.report.add_unfiltered_results(len(obj_list)) self.report.add(report_entry) return obj_list
def api_filter_handler(self, resource_obj, filter_objs): """ """ data_set = None if not filter_objs: # build api call (no filters) default_request_object = resource_obj.default_request_object data_set = self.api_response_handler(resource_obj, default_request_object) else: # # process each filter added to the resource object for retrieve # first_run = True # # each resource object can have x filter objects with an operator to join or intersect results # for filter_obj in filter_objs: obj_list = [] # temp storage for results on individual filter objects owners = filter_obj.owners if len(owners) == 0: # handle filters with no owners owners = [self._api_org] # use default org # iterate through all owners for o in owners: self.tcl.debug('owner: {0!s}'.format(o)) if len(filter_obj) > 0: # request object are for api filters for ro in filter_obj: if ro.owner_allowed: ro.set_owner(o) results = self.api_response_handler(resource_obj, ro) if ro.resource_type not in [ResourceType.OWNERS, ResourceType.VICTIMS, ResourceType.BATCH_JOBS]: # TODO: should this be done? # post filter owners for obj in results: if obj.owner_name.upper() != o.upper(): results.remove(obj) obj_list.extend(results) else: ro = filter_obj.default_request_object if ro.owner_allowed: ro.set_owner(o) results = self.api_response_handler(resource_obj, ro) if ro.resource_type not in [ResourceType.OWNERS, ResourceType.VICTIMS]: # TODO: should this be done? # post filter owners for obj in results: if obj.owner_name != o: results.remove(obj) obj_list.extend(results) # # post filters # pf_obj_set = set(obj_list) self.tcl.debug('count before post filter: {0:d}'.format(len(obj_list))) for pfo in filter_obj.post_filters: self.tcl.debug('pfo: {0!s}'.format(pfo)) # # Report Entry # report_entry = ReportEntry() report_entry.add_post_filter_object(pfo) # current post filter method filter_method = getattr(resource_obj, pfo.method) # current post filter results post_filter_results = set(filter_method(pfo.filter, pfo.operator, pfo.description)) pf_obj_set = pf_obj_set.intersection(post_filter_results) self.report.add(report_entry) # set obj_list to post_filter results if filter_obj.post_filters_len > 0: obj_list = list(pf_obj_set) self.tcl.debug('count after post filter: {0:d}'.format(len(obj_list))) # no need to join or intersect on first run if first_run: data_set = set(obj_list) first_run = False continue # # depending on the filter type the result will be intersected or joined # if filter_obj.operator is FilterSetOperator.AND: data_set = data_set.intersection(obj_list) elif filter_obj.operator is FilterSetOperator.OR: data_set.update(set(obj_list)) # # only add to report if these results should be tracked (exclude attribute, tags, etc) # self.report.add_filtered_results(len(data_set)) # # after intersection or join add the objects to the resource object # for obj in data_set: resource_obj.add_obj(obj)