def victim_associations(self):
""" retrieve associations for this group. associations are not stored within the object """
prop = self._resource_properties['association_victims']
ro = RequestObject()
ro.set_description('retrieve victim associations for {0}'.format(self._name))
ro.set_http_method(prop['http_method'])
ro.set_owner_allowed(prop['owner_allowed'])
ro.set_owner(self.owner_name)
ro.set_request_uri(prop['uri'].format(self._id))
ro.set_resource_pagination(prop['pagination'])
ro.set_resource_type(self._resource_type)
for item in self._tc.result_pagination(ro, 'victim'):
yield parse_victim(item, api_filter=ro.description, request_uri=ro.request_uri)
def api_response_handler(self, resource_obj, ro):
""" """
#
# initialize vars
#
api_response_dict = {}
obj_list = []
# only track filter counts on request from this method
ro.enable_track()
#
# debug
#
self.tcl.debug('Results Limit: {0!s}'.format(self._api_result_limit))
# only resource supports pagination
if ro.resource_pagination:
ro.set_result_limit(self._api_result_limit)
ro.set_result_start(0)
else:
ro.set_remaining_results(1)
while ro.remaining_results > 0:
#
# api request
#
api_response = self.api_request(ro)
# self.tcl.debug('Results Content: {0!s}'.format(api_response.content))
self.tcl.debug('Status Code: {0!s}'.format(api_response.status_code))
self.tcl.debug('Content Type: {0!s}'.format(api_response.headers['content-type']))
#
# Process API response
#
if api_response.headers['content-type'] == 'application/json':
api_response_dict = api_response.json()
# self.print_mem('after building dict')
# try and free memory for next api request
api_response.close()
del api_response # doesn't appear to clear memory
#
# BULK INDICATOR (does not have status)
#
if 'indicator' in api_response_dict:
if ro.resource_type == ResourceType.INDICATORS:
data = api_response_dict['indicator']
for item in data:
obj_list.append(parse_indicator(
item, resource_obj, ro.description, ro.request_uri, self._indicators_regex))
if len(obj_list) % 500 == 0:
self.tcl.debug('obj_list len: {0!s}'.format(len(obj_list)))
# self.print_mem('bulk process - {0:d} objects'.format(len(obj_list)))
elif api_response_dict['status'] == 'Failure':
# handle failed request (404 Resource not Found)
if 'message' in api_response_dict:
self.tcl.error('{0!s} "{1!s}"'.format(api_response_dict['message'], ro.description))
ro.set_remaining_results(0)
continue
#
# ADVERSARIES
#
elif ro.resource_type == ResourceType.ADVERSARIES:
data = api_response_dict['data']['adversary']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
obj_list.append(
parse_group(item, ResourceType.ADVERSARIES, resource_obj, ro.description, ro.request_uri))
#
# INDICATORS
#
elif ro.resource_type == ResourceType.INDICATORS:
data = api_response_dict['data']['indicator']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
obj_list.append(parse_indicator(
item, resource_obj, ro.description, ro.request_uri, self._indicators_regex))
#
# ADDRESSES
#
elif ro.resource_type == ResourceType.ADDRESSES:
data = api_response_dict['data']['address']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
obj_list.append(parse_indicator(
item, resource_obj, ro.description, ro.request_uri, self._indicators_regex))
#
# DOCUMENTS
#
elif ro.resource_type == ResourceType.DOCUMENTS:
data = api_response_dict['data']['document']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
obj_list.append(
parse_group(
item, ResourceType.DOCUMENTS, resource_obj, ro.description, ro.request_uri))
#
# EMAILS
#
elif ro.resource_type == ResourceType.EMAILS:
data = api_response_dict['data']['email']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
obj_list.append(
parse_group(
item, ResourceType.EMAILS, resource_obj, ro.description, ro.request_uri))
#
# EMAIL ADDRESSES
#
elif ro.resource_type == ResourceType.EMAIL_ADDRESSES:
data = api_response_dict['data']['emailAddress']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
obj_list.append(parse_indicator(
item, resource_obj, ro.description, ro.request_uri, self._indicators_regex))
#
# GROUPS
#
elif ro.resource_type == ResourceType.GROUPS:
data = api_response_dict['data']['group']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
obj_list.append(
parse_group(item, ResourceType.GROUPS, resource_obj, ro.description, ro.request_uri))
#
# FILES
#
elif ro.resource_type == ResourceType.FILES:
data = api_response_dict['data']['file']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
obj_list.append(parse_indicator(
item, resource_obj, ro.description, ro.request_uri, self._indicators_regex))
#
# HOSTS
#
elif ro.resource_type == ResourceType.HOSTS:
data = api_response_dict['data']['host']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
obj_list.append(parse_indicator(
item, resource_obj, ro.description, ro.request_uri, self._indicators_regex))
#
# DNSResolutions
#
elif ro.resource_type == ResourceType.DNS_RESOLUTIONS:
data = api_response_dict['data']['dnsResolution']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
if 'addresses' in item: # don't process dns resolutions that have no addresses
obj_list.append(parse_dns_resolution(item))
#
# INCIDENTS
#
elif ro.resource_type == ResourceType.INCIDENTS:
data = api_response_dict['data']['incident']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
obj_list.append(
parse_group(item, ResourceType.INCIDENTS, resource_obj, ro.description, ro.request_uri))
#
# OWNERS
#
elif ro.resource_type == ResourceType.OWNERS:
data = api_response_dict['data']['owner']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
obj_list.append(
parse_owner(item, resource_obj, ro.description, ro.request_uri))
#
# SIGNATURES
#
elif ro.resource_type == ResourceType.SIGNATURES:
data = api_response_dict['data']['signature']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
obj_list.append(
parse_group(item, ResourceType.SIGNATURES, resource_obj, ro.description, ro.request_uri))
#
# THREATS
#
elif ro.resource_type == ResourceType.THREATS:
data = api_response_dict['data']['threat']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
obj_list.append(
parse_group(item, ResourceType.THREATS, resource_obj, ro.description, ro.request_uri))
#
# URLS
#
elif ro.resource_type == ResourceType.URLS:
data = api_response_dict['data']['url']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
obj_list.append(parse_indicator(
item, resource_obj, ro.description, ro.request_uri, self._indicators_regex))
#
# VICTIMS
#
elif ro.resource_type == ResourceType.VICTIMS:
data = api_response_dict['data']['victim']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
# victims data comes back with no owner, manually add owner here
item['owner'] = ro.owner
obj_list.append(parse_victim(item, resource_obj, ro.description, ro.request_uri))
#
# BatchJobs
#
elif ro.resource_type == ResourceType.BATCH_JOBS:
data = api_response_dict['data']['batchStatus']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
# victims data comes back with no owner, manually add owner here
item['owner'] = ro.owner
obj_list.append(parse_batch_job(item, resource_obj, ro.description, ro.request_uri))
#
# memory testing
#
# self.print_mem('pagination - {0:d} objects'.format(len(obj_list)))
elif api_response.headers['content-type'] == 'text/plain':
self.tcl.error('{0!s} "{1!s}"'.format(api_response.content, ro.description))
ro.set_remaining_results(0)
continue
# add_obj resource_pagination if required
if ro.resource_pagination:
# get the number of results returned by the api
if ro.result_start == 0:
ro.set_remaining_results(api_response_dict['data']['resultCount'] - ro.result_limit)
else:
ro.set_remaining_results(ro.remaining_results - ro.result_limit)
# increment the start position
ro.set_result_start(ro.result_start + ro.result_limit)
else:
ro.set_remaining_results(0)
self.tcl.debug('Result Count: {0!s}'.format(len(obj_list)))
self.report.add_unfiltered_results(len(obj_list))
return obj_list
def api_response_handler(self, resource_obj, ro):
""" """
#
# initialize vars
#
api_response_dict = {}
obj_list = []
# only track filter counts on request from this method
ro.enable_track()
#
# debug
#
self.tcl.debug('Results Limit: {0!s}'.format(self._api_result_limit))
# only resource supports pagination
if ro.resource_pagination:
ro.set_result_limit(self._api_result_limit)
ro.set_result_start(0)
while ro.remaining_results > 0:
#
# api request
#
api_response = self.api_request(ro)
# self.tcl.debug('Results Content: {0!s}'.format(api_response.content))
self.tcl.debug('Status Code: {0!s}'.format(api_response.status_code))
self.tcl.debug('Content Type: {0!s}'.format(api_response.headers['content-type']))
#
# Process API response
#
if api_response.headers['content-type'] == 'application/json':
api_response_dict = api_response.json()
# self.print_mem('after building dict')
# try and free memory for next api request
api_response.close()
del api_response # doesn't appear to clear memory
#
# BULK INDICATOR (does not have status)
#
if 'indicator' in api_response_dict:
if ro.resource_type == ResourceType.INDICATORS:
data = api_response_dict['indicator']
for item in data:
obj_list.append(parse_indicator(item, resource_obj, ro.description, ro.request_uri))
if len(obj_list) % 500 == 0:
self.tcl.debug('obj_list len: {0!s}'.format(len(obj_list)))
self.print_mem('bulk process - {0:d} objects'.format(len(obj_list)))
elif api_response_dict['status'] == 'Failure':
# handle failed request (404 Resource not Found)
if 'message' in api_response_dict:
self.tcl.error('{0!s} "{1!s}"'.format(api_response_dict['message'], ro.description))
ro.set_remaining_results(0)
continue
#
# ADVERSARIES
#
elif ro.resource_type == ResourceType.ADVERSARIES:
data = api_response_dict['data']['adversary']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
obj_list.append(
parse_group(item, ResourceType.ADVERSARIES, resource_obj, ro.description, ro.request_uri))
#
# INDICATORS
#
elif ro.resource_type == ResourceType.INDICATORS:
data = api_response_dict['data']['indicator']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
obj_list.append(parse_indicator(item, resource_obj, ro.description, ro.request_uri))
#
# ADDRESSES
#
elif ro.resource_type == ResourceType.ADDRESSES:
data = api_response_dict['data']['address']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
obj_list.append(parse_indicator(item, resource_obj, ro.description, ro.request_uri))
#
# DOCUMENTS
#
elif ro.resource_type == ResourceType.DOCUMENTS:
data = api_response_dict['data']['document']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
obj_list.append(
parse_group(item, ResourceType.DOCUMENTS, resource_obj, ro.description, ro.request_uri))
#
# EMAILS
#
elif ro.resource_type == ResourceType.EMAILS:
data = api_response_dict['data']['email']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
obj_list.append(
parse_group(item, ResourceType.EMAILS, resource_obj, ro.description, ro.request_uri))
#
# EMAIL ADDRESSES
#
elif ro.resource_type == ResourceType.EMAIL_ADDRESSES:
data = api_response_dict['data']['emailAddress']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
obj_list.append(parse_indicator(item, resource_obj, ro.description, ro.request_uri))
#
# GROUPS
#
elif ro.resource_type == ResourceType.GROUPS:
data = api_response_dict['data']['group']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
obj_list.append(
parse_group(item, ResourceType.GROUPS, resource_obj, ro.description, ro.request_uri))
#
# FILES
#
elif ro.resource_type == ResourceType.FILES:
data = api_response_dict['data']['file']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
obj_list.append(parse_indicator(item, resource_obj, ro.description, ro.request_uri))
#
# HOSTS
#
elif ro.resource_type == ResourceType.HOSTS:
data = api_response_dict['data']['host']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
obj_list.append(parse_indicator(item, resource_obj, ro.description, ro.request_uri))
#
# DNSResolutions
#
elif ro.resource_type == ResourceType.DNS_RESOLUTIONS:
data = api_response_dict['data']['dnsResolution']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
if 'addresses' in item: # don't process dns resolutions that have no addresses
obj_list.append(parse_dns_resolution(item))
#
# INCIDENTS
#
elif ro.resource_type == ResourceType.INCIDENTS:
data = api_response_dict['data']['incident']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
obj_list.append(
parse_group(item, ResourceType.INCIDENTS, resource_obj, ro.description, ro.request_uri))
#
# OWNERS
#
elif ro.resource_type == ResourceType.OWNERS:
data = api_response_dict['data']['owner']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
obj_list.append(
parse_owner(item, resource_obj, ro.description, ro.request_uri))
#
# SIGNATURES
#
elif ro.resource_type == ResourceType.SIGNATURES:
data = api_response_dict['data']['signature']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
obj_list.append(
parse_group(item, ResourceType.SIGNATURES, resource_obj, ro.description, ro.request_uri))
#
# THREATS
#
elif ro.resource_type == ResourceType.THREATS:
data = api_response_dict['data']['threat']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
obj_list.append(
parse_group(item, ResourceType.THREATS, resource_obj, ro.description, ro.request_uri))
#
# URLS
#
elif ro.resource_type == ResourceType.URLS:
data = api_response_dict['data']['url']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
obj_list.append(parse_indicator(item, resource_obj, ro.description, ro.request_uri))
#
# VICTIMS
#
elif ro.resource_type == ResourceType.VICTIMS:
data = api_response_dict['data']['victim']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
# victims data comes back with no owner, manually add owner here
item['owner'] = ro.owner
obj_list.append(parse_victim(item, resource_obj, ro.description, ro.request_uri))
#
# BatchJobs
#
elif ro.resource_type == ResourceType.BATCH_JOBS:
data = api_response_dict['data']['batchStatus']
if not isinstance(data, list):
data = [data] # for single results to be a list
for item in data:
# victims data comes back with no owner, manually add owner here
item['owner'] = ro.owner
obj_list.append(parse_batch_job(item, resource_obj, ro.description, ro.request_uri))
#
# memory testing
#
self.print_mem('pagination - {0:d} objects'.format(len(obj_list)))
elif api_response.headers['content-type'] == 'text/plain':
self.tcl.error('{0!s} "{1!s}"'.format(api_response.content, ro.description))
ro.set_remaining_results(0)
continue
# add_obj resource_pagination if required
if ro.resource_pagination:
# get the number of results returned by the api
if ro.result_start == 0:
ro.set_remaining_results(api_response_dict['data']['resultCount'] - ro.result_limit)
else:
ro.set_remaining_results(ro.remaining_results - ro.result_limit)
# increment the start position
ro.set_result_start(ro.result_start + ro.result_limit)
else:
ro.set_remaining_results(0)
self.tcl.debug('Result Count: {0!s}'.format(len(obj_list)))
self.report.add_unfiltered_results(len(obj_list))
return obj_list