def _request_wrapper(self, recursed=False, **kwargs):
"""Wraps all requests for an AMP leaf in order to handle AMP specifics.
This should only be called by internal methods.
*Parameters*
recursed: boolean, keyword, default=False
Signals if this is the top level call.
\*\*kwargs: dictionary
Used to pass through arguments to wrapped methods.
"""
if not recursed:
for i in range(1, self.rpm_retries):
response_dict, status, include_filtered, exclude_filtered, cache_hit = \
self._request_wrapper(recursed=True, **kwargs)
if response_dict['status_code'] == 429 and not exclude_filtered:
# Pop the url from the responses_dict so we don't trigger a cache hit on the next request...
self.responses_dict.pop(kwargs['url'])
if i == (self.rpm_retries - 1):
logger.error(
'Requests per minute code 429 retry count exceeded. Exiting...'
)
sys.exit()
logger.info(
'AMP reports requests per minute exceeding 120. Sleeping for self._backoff_timer seconds...'
)
time.sleep(self.backoff_timer)
else:
next_link = tree_helpers.get_jsonpath_values(
self.next_link_query, response_dict)
next_url = None
if next_link:
next_url = next_link[0][0]
response_dict['next_link'] = next_url
else:
next_url = None
return response_dict, status, include_filtered, exclude_filtered, cache_hit, next_url
else:
return tree_helpers.process_json_request(**kwargs)
def process_base_url(url_so_far, base_url, query_dict):
print('processing url', url_so_far, base_url)
url = url_so_far
url_parts = base_url.split('/')
for i in range(0, len(url_parts)):
url_part = url_parts[i]
if url_part.startswith('$'):
query_responses = {}
if url not in query_dict:
query_url = self._prepare_url(url=url, params=params)
query_responses = self.get_all_items(
url=query_url, responses_dict=query_responses)
query_dict[url] = query_responses
else:
query_responses = query_dict[url]
query_path = '{}/{}'.format(url, url_part)
if query_path not in query_dict:
print(url_part, file=sys.stderr)
substitutes = tree_helpers.get_jsonpath_values(
url_part, query_responses)
if not substitutes:
logger.info(
'No substitute values found for free form query %s in url %s'
% (url_part, base_url))
return False
else:
query_dict[query_path] = substitutes
if url.endswith('/'):
url = url[:-1]
for substitute in substitutes:
complete = process_base_url(
'{}/{}/'.format(url, substitute),
'/'.join(url_parts[i + 1:]), query_dict)
if not complete:
return False
return True
else:
url += url_part + '/'
# Last part reached, get the responses...
url = self._prepare_url(url=url, params=params)
query_responses = self.get_all_items(url=url, responses_dict={})
query_dict[url] = query_responses
return query_responses
def object_dump(_dict=None, tab_string='', file=sys.stdout, responses_only=False):
"""Creates an outline of '_dict' to 'file'.
"""
def utf_print(_str, file=None):
print(_str, file=file)
def process_val(val, tab_string):
if type(val) is dict:
process_dict(val, tab_string)
elif type(val) is list:
process_list(val, tab_string)
else:
if type(val) is str and '-----BEGIN ' in val:
val = 'Certificate Encoding Omitted...'
utf_print(tab_string + str(val), file=file)
def process_list(_list, tab_string):
utf_print(tab_string + 'list', file=file)
tab_string += '\t'
list_counter = 0
for val in _list:
if type(val) is dict:
list_counter += 1
utf_print(tab_string + str(list_counter), file=file)
process_val(val, tab_string + '\t')
else:
process_val(val, tab_string)
def process_dict(_dict, tab_string):
utf_print(tab_string + 'dict', file=file)
tab_string += '\t'
for key, val in _dict.items():
utf_print(tab_string + key, file=file)
process_val(val, tab_string + '\t')
# file = codecs.open('text.txt', 'w', 'utf-8')
# sys.stdout = codecs.getwriter("utf-8")(sys.stdout.detach())
if responses_only:
_dict = get_jsonpath_values('$..json_dict', _dict)
outline_str = ''
process_val(_dict, tab_string)
def walk_samples(self,
sample_id_paths=None,
params=None,
responses_dict=None):
"""Walks threatgrid samples by id.
Returns: a responses dictionary
*Parameters*
sample_id_paths: dictionary, keyword, default=None
Defines the paths to retrieve for each id.
params: dictionary, keyword, default=None
Defines the search scope parameters (e.g. before=<strftime>).
responses_dict: dictionary, keyword, default=None
Allows the caller to override the default behavior to store responses in the self.responses_dict. Useful
if caller would like to keep the responses isolated.
"""
sample_path = 'api/v2/samples'
url = self._prepare_url(url=sample_path, params=params)
responses = self.get_all_items(url=url)
ids = tree_helpers.get_jsonpath_values('$..data.items[*].id',
responses)
if responses_dict is None:
responses_dict = self.responses_dict
else:
responses_dict = responses_dict
for id in ids:
url = sample_path + '/' + str(id)
for sample_id_path in sample_id_paths:
url = url + '/' + sample_id_path
url = self._prepare_url(url)
print(url)
self.GET_API_path(url=url, responses_dict=responses_dict)
return responses_dict
def get_type_and_id_tuple_list(base_path, json_dict):
type_list = tree_helpers.get_jsonpath_values(base_path + '.type',
json_dict)
id_list = tree_helpers.get_jsonpath_values(base_path + '.id', json_dict)
return zip(type_list, id_list)
def get_all_reference_dicts(FMC_instance):
def add_paths_to_dict(path_parts, paths_dict, API_path_keywords_list):
if path_parts:
path_part = path_parts.pop(0)
if not path_part in paths_dict:
paths_dict[path_part] = {}
if not path_part in API_path_keywords_list and not path_part.startswith(
'{'):
API_path_keywords_list.append(path_part)
paths_dict = OrderedDict(sorted(paths_dict.items()))
add_paths_to_dict(path_parts, paths_dict[path_part],
API_path_keywords_list)
else:
return
paths_dict = {}
resource_path_dict = {}
operations_dict = {}
model_path_dict = {}
path_model_dict = {}
models_dict = {}
API_path_keywords_list = []
all_API_paths_list = []
# Get the base paths
resource_tuple_list = tree_helpers.get_jsonpath_full_paths_and_values(
'$.features.*.[*]', FMC_instance._json_dict)
for i in range(0, len(resource_tuple_list)):
resource_key = re.sub(r'features.|\[[0-9]+\]|\.', '',
resource_tuple_list[i][0])
resource_dict = resource_tuple_list[i][1]
base_path = resource_dict['basePath']
API_version = resource_dict['apiVersion']
# resource_root = FMC_instance._url_host + '/api' + base_path + '/' + API_version + '/domain/' + FMC_instance.FMC_domain_ID + '/'
resource_path_dict[resource_key] = ''
models_list = tree_helpers.get_jsonpath_full_paths_and_values(
'$..models.*.id', resource_dict)
for model in models_list:
if not model is None:
model_path = model[0]
model_ID = model[1]
if not model_ID in models_dict:
properties_path = model_path
properties_path_list = properties_path.split('.')
properties_path_list.pop()
properties_path_list.append('properties')
properties_path = '.'.join(properties_path_list)
models_dict[model_ID] = tree_helpers.get_jsonpath_values(
properties_path, resource_dict)[0]
operations_list = tree_helpers.get_jsonpath_values(
'$..operations', resource_dict)
for operation_list in operations_list:
model_ID = None
for operation in operation_list:
if not operation is None:
path = re.sub('^/', '', operation['path'])
method = tree_helpers.get_jsonpath_values(
'$..method', operation)[0]
if model_ID is None:
for example in operation['examples']:
if not 'override' in example['url']:
if 'responseData' in example:
if 'type' in example['responseData']:
model_ID = example['responseData'][
'type']
break
if not path in operations_dict:
operations_dict[path] = {
'methods': {},
'operation_path': operation['path']
}
operations_dict[path]['methods'][method] = operation
if not model_ID:
model_ID = operation_list[-1]['type']['modelId']
# model_path_dict[model_ID] = path
path = operation_list[-1]['path']
path = re.sub('^/', '', path)
model_path_dict[model_ID] = path
path_model_dict[path] = model_ID
for path in operations_dict.keys():
operation_path = operations_dict[path]['operation_path']
all_API_paths_list.append(path)
path_parts = operation_path.split('/')
path_parts.pop(0) # Get rid of the first '/'
resource_path = path_parts[0]
if resource_path in resource_path_dict:
base_path = resource_path_dict[resource_path]
else:
continue
add_paths_to_dict(path_parts, paths_dict, API_path_keywords_list)
all_API_paths_list.sort()
return OrderedDict(sorted(resource_path_dict.items())), OrderedDict(sorted(operations_dict.items())), \
OrderedDict(sorted(model_path_dict.items())), OrderedDict(sorted(paths_dict.items())), \
API_path_keywords_list, all_API_paths_list, path_model_dict, models_dict
def create_outline(_dict=None,tab_string='',file=sys.stdout,smart_labels=True,responses_only=False):
"""Creates an outline of '_dict' to 'file'.
"""
def get_list_item_label(_dict):
if 'name' in _dict:
if type(_dict['name']) is str:
return 'name', _dict['name']
elif 'models' in _dict:
if type(_dict['models']) is str:
return 'models', _dict['models']
elif type(_dict['models']) is dict:
return 'models', list(_dict['models'].keys())[0]
elif 'content-type' in _dict:
if type(_dict['content-type']) is str:
return 'content-type', _dict['content-type']
elif type(_dict['models']) is dict:
return 'content-type', list(_dict['content-type'].keys())[0]
for key, val in _dict.items():
if type(val) is str:
if re.match(r'^[a-zA-Z]+$',val):
if re.match(r'^[A-Z]+$',val) or \
re.match(r'^[A-Z][a-z]+[A-Z][a-z]+$',val) or \
re.match(r'^[a-z]+[A-Z][a-z]+$',val):
return key, val
return None
def process_val(val, tab_string):
if type(val) is dict:
process_dict(val,tab_string)
elif type(val) is list:
process_list(val,tab_string)
else:
if type(val) is str and '-----BEGIN ' in val:
val = 'Certificate Encoding Omitted...'
print(tab_string+str(val),file=file)
def process_list(_list, tab_string):
print(tab_string+'list',file=file)
tab_string += '\t'
list_counter = 0
for val in _list:
if type(val) is dict:
if smart_labels:
label = get_list_item_label(val)
if not label is None:
print(tab_string+label[0]+'='+label[1],file=file)
else:
list_counter+=1
print(tab_string+str(list_counter),file=file)
else:
list_counter+=1
print(tab_string+str(list_counter),file=file)
process_val(val,tab_string + '\t')
else:
process_val(val,tab_string)
def process_dict(_dict,tab_string):
print(tab_string+'dict',file=file)
tab_string += '\t'
for key, val in _dict.items():
print(tab_string+key,file=file)
process_val(val,tab_string+'\t')
if responses_only:
_dict = get_jsonpath_values('$..json_dict',_dict)
outline_str = ''
process_val(_dict,tab_string)