def _request_wrapper(self, recursed=False, **kwargs): """Wraps all requests for an AMP leaf in order to handle AMP specifics. This should only be called by internal methods. *Parameters* recursed: boolean, keyword, default=False Signals if this is the top level call. \*\*kwargs: dictionary Used to pass through arguments to wrapped methods. """ if not recursed: for i in range(1, self.rpm_retries): response_dict, status, include_filtered, exclude_filtered, cache_hit = \ self._request_wrapper(recursed=True, **kwargs) if response_dict['status_code'] == 429 and not exclude_filtered: # Pop the url from the responses_dict so we don't trigger a cache hit on the next request... self.responses_dict.pop(kwargs['url']) if i == (self.rpm_retries - 1): logger.error( 'Requests per minute code 429 retry count exceeded. Exiting...' ) sys.exit() logger.info( 'AMP reports requests per minute exceeding 120. Sleeping for self._backoff_timer seconds...' ) time.sleep(self.backoff_timer) else: next_link = tree_helpers.get_jsonpath_values( self.next_link_query, response_dict) next_url = None if next_link: next_url = next_link[0][0] response_dict['next_link'] = next_url else: next_url = None return response_dict, status, include_filtered, exclude_filtered, cache_hit, next_url else: return tree_helpers.process_json_request(**kwargs)
def process_base_url(url_so_far, base_url, query_dict): print('processing url', url_so_far, base_url) url = url_so_far url_parts = base_url.split('/') for i in range(0, len(url_parts)): url_part = url_parts[i] if url_part.startswith('$'): query_responses = {} if url not in query_dict: query_url = self._prepare_url(url=url, params=params) query_responses = self.get_all_items( url=query_url, responses_dict=query_responses) query_dict[url] = query_responses else: query_responses = query_dict[url] query_path = '{}/{}'.format(url, url_part) if query_path not in query_dict: print(url_part, file=sys.stderr) substitutes = tree_helpers.get_jsonpath_values( url_part, query_responses) if not substitutes: logger.info( 'No substitute values found for free form query %s in url %s' % (url_part, base_url)) return False else: query_dict[query_path] = substitutes if url.endswith('/'): url = url[:-1] for substitute in substitutes: complete = process_base_url( '{}/{}/'.format(url, substitute), '/'.join(url_parts[i + 1:]), query_dict) if not complete: return False return True else: url += url_part + '/' # Last part reached, get the responses... url = self._prepare_url(url=url, params=params) query_responses = self.get_all_items(url=url, responses_dict={}) query_dict[url] = query_responses return query_responses
def object_dump(_dict=None, tab_string='', file=sys.stdout, responses_only=False): """Creates an outline of '_dict' to 'file'. """ def utf_print(_str, file=None): print(_str, file=file) def process_val(val, tab_string): if type(val) is dict: process_dict(val, tab_string) elif type(val) is list: process_list(val, tab_string) else: if type(val) is str and '-----BEGIN ' in val: val = 'Certificate Encoding Omitted...' utf_print(tab_string + str(val), file=file) def process_list(_list, tab_string): utf_print(tab_string + 'list', file=file) tab_string += '\t' list_counter = 0 for val in _list: if type(val) is dict: list_counter += 1 utf_print(tab_string + str(list_counter), file=file) process_val(val, tab_string + '\t') else: process_val(val, tab_string) def process_dict(_dict, tab_string): utf_print(tab_string + 'dict', file=file) tab_string += '\t' for key, val in _dict.items(): utf_print(tab_string + key, file=file) process_val(val, tab_string + '\t') # file = codecs.open('text.txt', 'w', 'utf-8') # sys.stdout = codecs.getwriter("utf-8")(sys.stdout.detach()) if responses_only: _dict = get_jsonpath_values('$..json_dict', _dict) outline_str = '' process_val(_dict, tab_string)
def walk_samples(self, sample_id_paths=None, params=None, responses_dict=None): """Walks threatgrid samples by id. Returns: a responses dictionary *Parameters* sample_id_paths: dictionary, keyword, default=None Defines the paths to retrieve for each id. params: dictionary, keyword, default=None Defines the search scope parameters (e.g. before=<strftime>). responses_dict: dictionary, keyword, default=None Allows the caller to override the default behavior to store responses in the self.responses_dict. Useful if caller would like to keep the responses isolated. """ sample_path = 'api/v2/samples' url = self._prepare_url(url=sample_path, params=params) responses = self.get_all_items(url=url) ids = tree_helpers.get_jsonpath_values('$..data.items[*].id', responses) if responses_dict is None: responses_dict = self.responses_dict else: responses_dict = responses_dict for id in ids: url = sample_path + '/' + str(id) for sample_id_path in sample_id_paths: url = url + '/' + sample_id_path url = self._prepare_url(url) print(url) self.GET_API_path(url=url, responses_dict=responses_dict) return responses_dict
def get_type_and_id_tuple_list(base_path, json_dict): type_list = tree_helpers.get_jsonpath_values(base_path + '.type', json_dict) id_list = tree_helpers.get_jsonpath_values(base_path + '.id', json_dict) return zip(type_list, id_list)
def get_all_reference_dicts(FMC_instance): def add_paths_to_dict(path_parts, paths_dict, API_path_keywords_list): if path_parts: path_part = path_parts.pop(0) if not path_part in paths_dict: paths_dict[path_part] = {} if not path_part in API_path_keywords_list and not path_part.startswith( '{'): API_path_keywords_list.append(path_part) paths_dict = OrderedDict(sorted(paths_dict.items())) add_paths_to_dict(path_parts, paths_dict[path_part], API_path_keywords_list) else: return paths_dict = {} resource_path_dict = {} operations_dict = {} model_path_dict = {} path_model_dict = {} models_dict = {} API_path_keywords_list = [] all_API_paths_list = [] # Get the base paths resource_tuple_list = tree_helpers.get_jsonpath_full_paths_and_values( '$.features.*.[*]', FMC_instance._json_dict) for i in range(0, len(resource_tuple_list)): resource_key = re.sub(r'features.|\[[0-9]+\]|\.', '', resource_tuple_list[i][0]) resource_dict = resource_tuple_list[i][1] base_path = resource_dict['basePath'] API_version = resource_dict['apiVersion'] # resource_root = FMC_instance._url_host + '/api' + base_path + '/' + API_version + '/domain/' + FMC_instance.FMC_domain_ID + '/' resource_path_dict[resource_key] = '' models_list = tree_helpers.get_jsonpath_full_paths_and_values( '$..models.*.id', resource_dict) for model in models_list: if not model is None: model_path = model[0] model_ID = model[1] if not model_ID in models_dict: properties_path = model_path properties_path_list = properties_path.split('.') properties_path_list.pop() properties_path_list.append('properties') properties_path = '.'.join(properties_path_list) models_dict[model_ID] = tree_helpers.get_jsonpath_values( properties_path, resource_dict)[0] operations_list = tree_helpers.get_jsonpath_values( '$..operations', resource_dict) for operation_list in operations_list: model_ID = None for operation in operation_list: if not operation is None: path = re.sub('^/', '', operation['path']) method = tree_helpers.get_jsonpath_values( '$..method', operation)[0] if model_ID is None: for example in operation['examples']: if not 'override' in example['url']: if 'responseData' in example: if 'type' in example['responseData']: model_ID = example['responseData'][ 'type'] break if not path in operations_dict: operations_dict[path] = { 'methods': {}, 'operation_path': operation['path'] } operations_dict[path]['methods'][method] = operation if not model_ID: model_ID = operation_list[-1]['type']['modelId'] # model_path_dict[model_ID] = path path = operation_list[-1]['path'] path = re.sub('^/', '', path) model_path_dict[model_ID] = path path_model_dict[path] = model_ID for path in operations_dict.keys(): operation_path = operations_dict[path]['operation_path'] all_API_paths_list.append(path) path_parts = operation_path.split('/') path_parts.pop(0) # Get rid of the first '/' resource_path = path_parts[0] if resource_path in resource_path_dict: base_path = resource_path_dict[resource_path] else: continue add_paths_to_dict(path_parts, paths_dict, API_path_keywords_list) all_API_paths_list.sort() return OrderedDict(sorted(resource_path_dict.items())), OrderedDict(sorted(operations_dict.items())), \ OrderedDict(sorted(model_path_dict.items())), OrderedDict(sorted(paths_dict.items())), \ API_path_keywords_list, all_API_paths_list, path_model_dict, models_dict
def create_outline(_dict=None,tab_string='',file=sys.stdout,smart_labels=True,responses_only=False): """Creates an outline of '_dict' to 'file'. """ def get_list_item_label(_dict): if 'name' in _dict: if type(_dict['name']) is str: return 'name', _dict['name'] elif 'models' in _dict: if type(_dict['models']) is str: return 'models', _dict['models'] elif type(_dict['models']) is dict: return 'models', list(_dict['models'].keys())[0] elif 'content-type' in _dict: if type(_dict['content-type']) is str: return 'content-type', _dict['content-type'] elif type(_dict['models']) is dict: return 'content-type', list(_dict['content-type'].keys())[0] for key, val in _dict.items(): if type(val) is str: if re.match(r'^[a-zA-Z]+$',val): if re.match(r'^[A-Z]+$',val) or \ re.match(r'^[A-Z][a-z]+[A-Z][a-z]+$',val) or \ re.match(r'^[a-z]+[A-Z][a-z]+$',val): return key, val return None def process_val(val, tab_string): if type(val) is dict: process_dict(val,tab_string) elif type(val) is list: process_list(val,tab_string) else: if type(val) is str and '-----BEGIN ' in val: val = 'Certificate Encoding Omitted...' print(tab_string+str(val),file=file) def process_list(_list, tab_string): print(tab_string+'list',file=file) tab_string += '\t' list_counter = 0 for val in _list: if type(val) is dict: if smart_labels: label = get_list_item_label(val) if not label is None: print(tab_string+label[0]+'='+label[1],file=file) else: list_counter+=1 print(tab_string+str(list_counter),file=file) else: list_counter+=1 print(tab_string+str(list_counter),file=file) process_val(val,tab_string + '\t') else: process_val(val,tab_string) def process_dict(_dict,tab_string): print(tab_string+'dict',file=file) tab_string += '\t' for key, val in _dict.items(): print(tab_string+key,file=file) process_val(val,tab_string+'\t') if responses_only: _dict = get_jsonpath_values('$..json_dict',_dict) outline_str = '' process_val(_dict,tab_string)