def api_request(self, ro):
""" """
api_response = None
fail_msg = None
h_content_length = None
h_content_type = None
start = datetime.now()
#
# enable activity log
#
if self._activity_log:
ro.enable_activity_log()
#
# prepare request
#
url = '{0!s}{1!s}'.format(self._api_url, ro.request_uri)
api_request = Request(ro.http_method, url, data=ro.body, params=ro.payload)
request_prepped = api_request.prepare()
#
# generate headers
#
ro.set_path_url(request_prepped.path_url)
self._api_request_headers(ro)
request_prepped.prepare_headers(ro.headers)
#
# Debug
#
self.tcl.debug('request_object: {0!s}'.format(ro))
self.tcl.debug('url: {0!s}'.format(url))
self.tcl.debug('path url: {0!s}'.format(request_prepped.path_url))
#
# api request (gracefully handle temporary communications issues with the API)
#
for i in range(1, self._api_retries + 1, 1):
try:
api_response = self._session.send(
request_prepped, verify=self._verify_ssl, timeout=self._api_request_timeout,
proxies=self._proxies, stream=False)
break
except exceptions.ReadTimeout as e:
self.tcl.error('Error: {0!s}'.format(e))
self.tcl.error('The server may be experiencing delays at the moment.')
self.tcl.info('Pausing for {0!s} seconds to give server time to catch up.'.format(self._api_sleep))
time.sleep(self._api_sleep)
self.tcl.info('Retry {0!s} ....'.format(i))
if i == self._api_retries:
self.tcl.critical('Exiting: {0!s}'.format(e))
raise RuntimeError(e)
except exceptions.ConnectionError as e:
self.tcl.error('Error: {0!s}'.format(e))
self.tcl.error('Connection Error. The server may be down.')
self.tcl.info('Pausing for {0!s} seconds to give server time to catch up.'.format(self._api_sleep))
time.sleep(self._api_sleep)
self.tcl.info('Retry {0!s} ....'.format(i))
if i == self._api_retries:
self.tcl.critical('Exiting: {0!s}'.format(e))
raise RuntimeError(e)
except socket.error as e:
self.tcl.critical('Exiting: {0!s}'.format(e))
raise RuntimeError(e)
#
# header values
#
if 'content-length' in api_response.headers:
h_content_length = api_response.headers['content-length']
if 'content-type' in api_response.headers:
h_content_type = api_response.headers['content-type']
#
# raise exception on *critical* errors
#
non_critical_errors = [
b'The MD5 for this File is invalid, a File with this MD5 already exists', # 400 (application/json)
b'The SHA-1 for this File is invalid, a File with this SHA-1 already exists', # 400 (application/json)
b'The SHA-256 for this File is invalid, a File with this SHA-256 already exists', # 400 (application/json)
b'The requested resource was not found', # 404 (application/json)
b'Could not find resource for relative', # 500 (text/plain)
b'The requested Security Label was not removed - access was denied', # 401 (application/json)
]
#
# TODO: work out some logic to improve the API error handling, possible area where API could improve
#
# valid status codes 200, 201, 202
# if api_response.status_code in [400, 401, 403, 500, 503]:
if api_response.status_code not in [200, 201, 202]:
# check for non critical errors that have bad status codes
nce_found = False
fail_msg = api_response.content
for nce in non_critical_errors:
# api_response_dict['message'] not in non_critical_errors:
if re.findall(nce, api_response.content):
nce_found = True
break
if ro.failure_callback is not None:
ro.failure_callback(api_response.status_code)
# raise error on bad status codes that are not defined as nce
if not nce_found:
self.tcl.critical('Status Code: {0:d}'.format(api_response.status_code))
self.tcl.critical('Failed API Response: {0!s}'.format(api_response.content))
if ro.failure_callback is not None:
ro.failure_callback(api_response.status_code)
raise RuntimeError(api_response.content)
#
# set response encoding (best guess)
#
if api_response.encoding is None:
api_response.encoding = api_response.apparent_encoding
#
# Debug
#
self.tcl.debug('url: %s', api_response.url)
self.tcl.debug('status_code: %s', api_response.status_code)
self.tcl.debug('content-length: %s', h_content_length)
self.tcl.debug('content-type: %s', h_content_type)
#
# Report
#
self.report.add_api_call() # count api calls
self.report.add_request_time(datetime.now() - start)
self.tcl.debug('Request Time: {0!s}'.format(datetime.now() - start))
if self._enable_report:
report_entry = ReportEntry()
report_entry.add_request_object(ro)
report_entry.set_request_url(api_response.url)
report_entry.set_status_code(api_response.status_code)
report_entry.set_failure_msg(fail_msg)
self.report.add(report_entry)
#
# return response
#
# self.print_mem('end _api_request')
return api_response
def api_filter_handler(self, resource_obj, filter_objs):
""" """
data_set = None
if not filter_objs:
# build api call (no filters)
default_request_object = resource_obj.default_request_object
data_set = self.api_response_handler(resource_obj, default_request_object)
else:
#
# process each filter added to the resource object for retrieve
#
first_run = True
#
# each resource object can have x filter objects with an operator to join or intersect results
#
for filter_obj in filter_objs:
obj_list = [] # temp storage for results on individual filter objects
owners = filter_obj.owners
if len(owners) == 0: # handle filters with no owners
owners = [self._api_org] # use default org
# iterate through all owners
for o in owners:
self.tcl.debug('owner: {0!s}'.format(o))
if len(filter_obj) > 0:
# request object are for api filters
for ro in filter_obj:
if ro.owner_allowed:
ro.set_owner(o)
results = self.api_response_handler(resource_obj, ro)
if ro.resource_type not in [ResourceType.OWNERS,
ResourceType.VICTIMS,
ResourceType.BATCH_JOBS]:
# TODO: should this be done?
# post filter owners
for obj in results:
if obj.owner_name.upper() != o.upper():
results.remove(obj)
obj_list.extend(results)
else:
ro = filter_obj.default_request_object
if ro.owner_allowed:
ro.set_owner(o)
results = self.api_response_handler(resource_obj, ro)
if ro.resource_type not in [ResourceType.OWNERS, ResourceType.VICTIMS]:
# TODO: should this be done?
# post filter owners
for obj in results:
if obj.owner_name.upper() != o.upper():
results.remove(obj)
obj_list.extend(results)
#
# post filters
#
pf_obj_set = set(obj_list)
self.tcl.debug('count before post filter: {0:d}'.format(len(obj_list)))
for pfo in filter_obj.post_filters:
self.tcl.debug('pfo: {0!s}'.format(pfo))
#
# Report Entry
#
report_entry = ReportEntry()
report_entry.add_post_filter_object(pfo)
# current post filter method
filter_method = getattr(resource_obj, pfo.method)
# current post filter results
post_filter_results = set(filter_method(pfo.filter, pfo.operator, pfo.description))
pf_obj_set = pf_obj_set.intersection(post_filter_results)
self.report.add(report_entry)
# set obj_list to post_filter results
if filter_obj.post_filters_len > 0:
obj_list = list(pf_obj_set)
self.tcl.debug('count after post filter: {0:d}'.format(len(obj_list)))
# no need to join or intersect on first run
if first_run:
data_set = set(obj_list)
first_run = False
continue
#
# depending on the filter type the result will be intersected or joined
#
if filter_obj.operator is FilterSetOperator.AND:
data_set = data_set.intersection(obj_list)
elif filter_obj.operator is FilterSetOperator.OR:
data_set.update(set(obj_list))
#
# only add to report if these results should be tracked (exclude attribute, tags, etc)
#
self.report.add_filtered_results(len(data_set))
#
# after intersection or join add the objects to the resource object
#
for obj in data_set:
resource_obj.add_obj(obj)
def api_request(self, ro):
""" """
api_response = None
fail_msg = None
h_content_length = None
h_content_type = None
start = datetime.now()
#
# enable activity log
#
# request_object.enable_activity_mode()
#
# prepare request
#
url = '{0!s}{1!s}'.format(self._api_url, ro.request_uri)
api_request = Request(ro.http_method, url, data=ro.body, params=ro.payload)
request_prepped = api_request.prepare()
#
# generate headers
#
ro.set_path_url(request_prepped.path_url)
self._api_request_headers(ro)
request_prepped.prepare_headers(ro.headers)
#
# Debug
#
self.tcl.debug('request_object: {0!s}'.format(ro))
self.tcl.debug('url: {0!s}'.format(url))
self.tcl.debug('path url: {0!s}'.format(request_prepped.path_url))
#
# api request (gracefully handle temporary communications issues with the API)
#
for i in range(1, self._api_retries + 1, 1):
try:
api_response = self._session.send(
request_prepped, verify=self._verify_ssl, timeout=self._api_request_timeout,
proxies=self._proxies, stream=False)
break
except exceptions.ReadTimeout as e:
self.tcl.error('Error: {0!s}'.format(e))
self.tcl.error('The server may be experiencing delays at the moment.')
self.tcl.info('Pausing for {0!s} seconds to give server time to catch up.'.format(self._api_sleep))
time.sleep(self._api_sleep)
self.tcl.info('Retry {0!s} ....'.format(i))
if i == self._api_retries:
self.tcl.critical('Exiting: {0!s}'.format(e))
raise RuntimeError(e)
except exceptions.ConnectionError as e:
self.tcl.error('Error: {0!s}'.format(e))
self.tcl.error('Connection Error. The server may be down.')
self.tcl.info('Pausing for {0!s} seconds to give server time to catch up.'.format(self._api_sleep))
time.sleep(self._api_sleep)
self.tcl.info('Retry {0!s} ....'.format(i))
if i == self._api_retries:
self.tcl.critical('Exiting: {0!s}'.format(e))
raise RuntimeError(e)
except socket.error as e:
self.tcl.critical('Exiting: {0!s}'.format(e))
raise RuntimeError(e)
#
# header values
#
if 'content-length' in api_response.headers:
h_content_length = api_response.headers['content-length']
if 'content-type' in api_response.headers:
h_content_type = api_response.headers['content-type']
#
# raise exception on *critical* errors
#
non_critical_errors = [
b'The MD5 for this File is invalid, a File with this MD5 already exists', # 400 (application/json)
b'The SHA-1 for this File is invalid, a File with this SHA-1 already exists', # 400 (application/json)
b'The SHA-256 for this File is invalid, a File with this SHA-256 already exists', # 400 (application/json)
b'The requested resource was not found', # 404 (application/json)
b'Could not find resource for relative', # 500 (text/plain)
b'The requested Security Label was not removed - access was denied', # 401 (application/json)
]
#
# TODO: work out some logic to improve the API error handling, possible area where API could improve
#
# valid status codes 200, 201, 202
# if api_response.status_code in [400, 401, 403, 500, 503]:
if api_response.status_code not in [200, 201, 202]:
# check for non critical errors that have bad status codes
nce_found = False
fail_msg = api_response.content
for nce in non_critical_errors:
# api_response_dict['message'] not in non_critical_errors:
if re.findall(nce, api_response.content):
nce_found = True
break
if ro.failure_callback is not None:
ro.failure_callback(api_response.status_code)
# raise error on bad status codes that are not defined as nce
if not nce_found:
self.tcl.critical('Status Code: {0:d}'.format(api_response.status_code))
self.tcl.critical('Failed API Response: {0!s}'.format(api_response.content))
if ro.failure_callback is not None:
ro.failure_callback(api_response.status_code)
raise RuntimeError(api_response.content)
#
# set response encoding (best guess)
#
if api_response.encoding is None:
api_response.encoding = api_response.apparent_encoding
#
# Debug
#
self.tcl.debug('url: %s', api_response.url)
self.tcl.debug('status_code: %s', api_response.status_code)
self.tcl.debug('content-length: %s', h_content_length)
self.tcl.debug('content-type: %s', h_content_type)
#
# Report
#
self.report.add_api_call() # count api calls
self.report.add_request_time(datetime.now() - start)
self.tcl.debug('Request Time: {0!s}'.format(datetime.now() - start))
if self._enable_report:
report_entry = ReportEntry()
report_entry.add_request_object(ro)
report_entry.set_request_url(api_response.url)
report_entry.set_status_code(api_response.status_code)
report_entry.set_failure_msg(fail_msg)
self.report.add(report_entry)
#
# return response
#
# self.print_mem('end _api_request')
return api_response
def api_build_request(self, resource_obj, request_object, owners=None):
""" """
#
# initialize vars
#
obj_list = []
if owners is None or not owners:
owners = [self._api_org] # set owners to default org
else:
owners = list(owners) # get copy of owners list for pop
count = len(owners)
modified_since = None
request_payload = {}
result_start = 0
result_remaining = 0
#
# resource object values
#
body = request_object.body
content_type = request_object.content_type
http_method = request_object.http_method
owner_allowed = request_object.owner_allowed
resource_pagination = request_object.resource_pagination
resource_type = request_object.resource_type
request_uri = request_object.request_uri
#
# ReportEntry (create a report entry for this request)
#
report_entry = ReportEntry()
report_entry.set_action(request_object.name)
report_entry.set_resource_type(resource_obj.resource_type)
report_entry.add_data({'HTTP Method': http_method})
report_entry.add_data({'Max Results': self._api_max_results})
report_entry.add_data({'Owners': str(owners)})
report_entry.add_data({'Owner Allowed': owner_allowed})
report_entry.add_data({'Request URI': request_uri})
report_entry.add_data({'Request Body': body})
report_entry.add_data({'Resource Pagination': resource_pagination})
report_entry.add_data({'Resource Type': resource_type})
#
# debug
#
self.tcl.debug('Action: {0}'.format(request_object.name))
self.tcl.debug('Resource Type: {0}'.format(resource_obj.resource_type))
self.tcl.debug('HTTP Method: {0}'.format(http_method))
self.tcl.debug('Max Results: {0}'.format(self._api_max_results))
self.tcl.debug('Owners: {0}'.format(str(owners)))
self.tcl.debug('Owner Allowed: {0}'.format(owner_allowed))
self.tcl.debug('Request URI: {0}'.format(request_uri))
self.tcl.debug('Request Body: {0}'.format(body))
self.tcl.debug('Resource Pagination: {0}'.format(resource_pagination))
self.tcl.debug('Resource Type: {0}'.format(resource_type))
# TODO: what would happen if this was always set to request object value?
if resource_type.name in [
'INDICATORS', 'ADDRESSES', 'EMAIL_ADDRESSES', 'FILES', 'HOSTS', 'URLS']:
# TODO: find a cleaner way
if not re.findall('bulk', request_uri):
modified_since = resource_obj.get_modified_since()
# update resource object with max results
# ???moved to report resource_obj.set_max_results(self._api_max_results)
# append uri to resource object
# ???moved to report resource_obj.add_uris(request_uri)
# iterate through all owners and results
if owner_allowed or resource_pagination:
# DEBUG
if modified_since is not None:
request_payload['modifiedSince'] = modified_since
# ReportEntry
report_entry.add_data({'Modified Since': modified_since})
# if request_object.owner_allowed:
# # if len(list(request_object.owners)) > 0:
# # owners = list(request_object.owners)
# count = len(owners)
for x in xrange(count):
retrieve_data = True
# only add_obj owner parameter if owners is allowed
if owner_allowed:
owner = owners.pop(0)
request_payload['owner'] = owner
# DEBUG
self.tcl.debug('owner: %s', owner)
self.tcl.debug('request_payload: %s', request_payload)
# only add_obj result parameters if resource_pagination is allowed
if resource_pagination:
result_limit = int(self._api_max_results)
result_remaining = result_limit
result_start = 0
while retrieve_data:
# set retrieve data to False to prevent loop for non paginating request
retrieve_data = False
# only add_obj result parameters if resource_pagination is allowed
if request_object.resource_pagination:
request_payload['resultLimit'] = result_limit
request_payload['resultStart'] = result_start
# DEBUG
self.tcl.debug('result_limit: %s', result_limit)
self.tcl.debug('result_start: %s', result_start)
#
# api request
#
api_response = self._api_request(
request_uri, request_payload=request_payload, http_method=http_method, body=body)
api_response.encoding = 'utf-8'
# ReportEntry
report_entry.set_status_code(api_response.status_code)
report_entry.add_request_url(api_response.url)
# break is status is not valid
if api_response.status_code not in [200, 201, 202]:
# ReportEntry
report_entry.set_status('Failure')
report_entry.set_status_code(api_response.status_code)
report_entry.add_data({'Failure Message': api_response.content})
# Logging
resource_obj.add_error_message(ErrorCodes.e80000.value.format(api_response.content))
break
#
# CSV Special Case
#
if re.findall('bulk/csv$', request_object.request_uri):
obj_list.extend(
self._api_process_response_csv(resource_obj, api_response.content))
break
#
# parse response
#
api_response_dict = api_response.json()
resource_obj.current_url = api_response.url
# update group object with api response data
resource_obj.add_api_response(api_response.content)
resource_obj.add_status_code(api_response.status_code)
# resource_obj.add_error_message(api_response.content)
#
# bulk indicators
#
# indicator response has no status so it must come first
if 'indicator' in api_response_dict:
#
# process response
#
obj_list.extend(self._api_process_response(
resource_obj, api_response, request_object))
#
# non Success status
#
elif api_response_dict['status'] != 'Success':
# ReportEntry
report_entry.set_status(api_response_dict['status'])
report_entry.add_data(
{'Failure Message': api_response_dict['message']})
#
# normal response
#
elif 'data' in api_response_dict:
# ReportEntry
report_entry.set_status(api_response_dict['status'])
# update resource object
resource_obj.add_status(ApiStatus[api_response_dict['status'].upper()])
#
# process response
#
obj_list.extend(self._api_process_response(
resource_obj, api_response, request_object))
# add_obj resource_pagination if required
if request_object.resource_pagination:
# get the number of results returned by the api
if result_start == 0:
result_remaining = api_response_dict['data']['resultCount']
result_remaining -= result_limit
# flip retrieve data flag if there are more results to pull
if result_remaining > 0:
retrieve_data = True
# increment the start position
result_start += result_limit
else:
resource_obj.add_error_message(api_response.content)
elif content_type == 'application/octet-stream':
#
# api request
#
api_response = self._api_request(
request_uri, request_payload={}, http_method=http_method,
body=body, content_type=content_type)
# ReportEntry
report_entry.set_status_code(api_response.status_code)
report_entry.add_request_url(api_response.url)
if api_response.status_code not in [200, 201, 202]:
# ReportEntry
report_entry.set_status('Failure')
report_entry.add_data({'Failure Message': api_response.content})
# Logging
self.tcl.critical(ErrorCodes.e80000.value.format(api_response.content))
raise RuntimeError(ErrorCodes.e90001.value)
else:
report_entry.set_status('Success')
return api_response.content
else:
#
# api request
#
api_response = self._api_request(
request_uri, request_payload={}, http_method=http_method, body=body)
api_response.encoding = 'utf-8'
if 'content-type' in api_response.headers:
content_type = api_response.headers['content-type']
# ReportData
report_entry.set_status_code(api_response.status_code)
report_entry.add_request_url(api_response.url)
# break is status is not valid
if api_response.status_code not in [200, 201, 202]:
if api_response.status_code == 404:
# failure_message = api_response.json()['message']
failure_message = api_response.content
else:
failure_message = api_response.content
# ReportEntry
report_entry.set_status('Failure')
report_entry.add_data({'Failure Message': failure_message})
# Logging
self.tcl.critical(ErrorCodes.e80000.value.format(api_response.content))
raise RuntimeError(ErrorCodes.e90001.value)
elif content_type == "text/plain":
# signature download
return api_response.content
else:
api_response_dict = api_response.json()
resource_obj.current_url = api_response.url
# ReportEntry
report_entry.set_status(api_response_dict['status'])
# update group object with api response data
resource_obj.add_api_response(api_response.content)
resource_obj.add_status_code(api_response.status_code)
resource_obj.add_status(ApiStatus[api_response_dict['status'].upper()])
# no need to process data for deletes or if no data exists
if http_method != 'DELETE' and 'data' in api_response_dict:
#
# process response
#
processed_data = self._api_process_response(
resource_obj, api_response, request_object)
obj_list.extend(processed_data)
# ReportData
report_entry.add_data({'Result Count': len(obj_list)})
# Report
self.report.add_unfiltered_results(len(obj_list))
self.report.add(report_entry)
return obj_list
def api_filter_handler(self, resource_obj, filter_objs):
""" """
data_set = None
if not filter_objs:
# build api call (no filters)
default_request_object = resource_obj.default_request_object
data_set = self.api_response_handler(resource_obj, default_request_object)
else:
#
# process each filter added to the resource object for retrieve
#
first_run = True
#
# each resource object can have x filter objects with an operator to join or intersect results
#
for filter_obj in filter_objs:
obj_list = [] # temp storage for results on individual filter objects
owners = filter_obj.owners
if len(owners) == 0: # handle filters with no owners
owners = [self._api_org] # use default org
# iterate through all owners
for o in owners:
self.tcl.debug('owner: {0!s}'.format(o))
if len(filter_obj) > 0:
# request object are for api filters
for ro in filter_obj:
if ro.owner_allowed:
ro.set_owner(o)
results = self.api_response_handler(resource_obj, ro)
if ro.resource_type not in [ResourceType.OWNERS,
ResourceType.VICTIMS,
ResourceType.BATCH_JOBS]:
# TODO: should this be done?
# post filter owners
for obj in results:
if obj.owner_name.upper() != o.upper():
results.remove(obj)
obj_list.extend(results)
else:
ro = filter_obj.default_request_object
if ro.owner_allowed:
ro.set_owner(o)
results = self.api_response_handler(resource_obj, ro)
if ro.resource_type not in [ResourceType.OWNERS, ResourceType.VICTIMS]:
# TODO: should this be done?
# post filter owners
for obj in results:
if obj.owner_name != o:
results.remove(obj)
obj_list.extend(results)
#
# post filters
#
pf_obj_set = set(obj_list)
self.tcl.debug('count before post filter: {0:d}'.format(len(obj_list)))
for pfo in filter_obj.post_filters:
self.tcl.debug('pfo: {0!s}'.format(pfo))
#
# Report Entry
#
report_entry = ReportEntry()
report_entry.add_post_filter_object(pfo)
# current post filter method
filter_method = getattr(resource_obj, pfo.method)
# current post filter results
post_filter_results = set(filter_method(pfo.filter, pfo.operator, pfo.description))
pf_obj_set = pf_obj_set.intersection(post_filter_results)
self.report.add(report_entry)
# set obj_list to post_filter results
if filter_obj.post_filters_len > 0:
obj_list = list(pf_obj_set)
self.tcl.debug('count after post filter: {0:d}'.format(len(obj_list)))
# no need to join or intersect on first run
if first_run:
data_set = set(obj_list)
first_run = False
continue
#
# depending on the filter type the result will be intersected or joined
#
if filter_obj.operator is FilterSetOperator.AND:
data_set = data_set.intersection(obj_list)
elif filter_obj.operator is FilterSetOperator.OR:
data_set.update(set(obj_list))
#
# only add to report if these results should be tracked (exclude attribute, tags, etc)
#
self.report.add_filtered_results(len(data_set))
#
# after intersection or join add the objects to the resource object
#
for obj in data_set:
resource_obj.add_obj(obj)
