def queue_bulk_threats(self, atom_list, payload): hashkey_created = [] bulk_in_flight = [] # bulk task uuid unchecked for batch in split_list(atom_list, self._batch_size()): if len(bulk_in_flight) >= self.OCD_DTL_MAX_BULK_THREATS_IN_FLIGHT: bulk_threat_task_uuid = bulk_in_flight.pop(0) hashkey_created += self.check_bulk_threats_added(bulk_threat_task_uuid) payload['atom_values'] = '\n'.join(batch) # Raw csv expected response = self.datalake_requests(self.url, 'post', self._post_headers(), payload) task_uid = response.get('task_uuid') if task_uid: bulk_in_flight.append(response['task_uuid']) else: logger.warning(f'batch of threats from {batch[0]} to {batch[-1]} failed to be created') # Finish to check the other bulk tasks for bulk_threat_task_uuid in bulk_in_flight: hashkey_created += self.check_bulk_threats_added(bulk_threat_task_uuid) nb_threats = len(hashkey_created) if nb_threats > 0: ok_sign = '\x1b[0;30;42m' + ' OK ' + '\x1b[0m' logger.info(f'Created {nb_threats} threats'.ljust(self.terminal_size - 6, ' ') + ok_sign) else: ko_sign = '\x1b[0;30;41m' + ' KO ' + '\x1b[0m' logger.info(f'Failed to create any threats'.ljust(self.terminal_size - 6, ' ') + ko_sign) return set(hashkey_created)
def main(override_args=None): starter = BaseScripts() logger.debug(f'START: get_query_hash.py') # Load initial args parser = starter.start('Retrieve a query hash from a query body (a json used for the Advanced Search).') required_named = parser.add_argument_group('required arguments') required_named.add_argument( 'query_body_path', help='path to the json file containing the query body', ) if override_args: args = parser.parse_args(override_args) else: args = parser.parse_args() # Load api_endpoints and tokens endpoint_config, main_url, tokens = starter.load_config(args) with open(args.query_body_path, 'r') as query_body_file: query_body = json.load(query_body_file) logger.debug(f'Retrieving query hash for query body: {query_body}') advanced_search = AdvancedSearch(endpoint_config, args.env, tokens) response = advanced_search.get_threats(query_body, limit=0) if not response or 'query_hash' not in response: logger.error("Couldn't retrieve a query hash, is the query body valid ?") exit(1) query_hash = response['query_hash'] if args.output: with open(args.output, 'w') as output: output.write(query_hash) logger.info(f'Query hash saved in {args.output}') else: logger.info(f'Query hash associated: {query_hash}')
def get_json(self, list_threats: list): """ Retrieve the JSON file of a list of threats and their comments. :return dict, list: threats found and a list of hashkey not found """ total_hash = len(list_threats) dict_threat = {'count': total_hash, 'results': []} list_of_lost_hashes = [] for index, threat in enumerate(list_threats): request_url = self.url + threat response_dict = self.datalake_requests( request_url, 'get', self._get_headers(), None, ) final_dict = response_dict if not response_dict.get('hashkey'): list_of_lost_hashes.append(threat) logger.info( f'{str(index).ljust(5)}:{threat.ljust(self.terminal_size - 11)}\x1b[0;30;41mERROR\x1b[0m' ) else: logger.info( f'{str(index).ljust(5)}:{threat.ljust(self.terminal_size - 10)}\x1b[0;30;42m OK \x1b[0m' ) response_dict = self.datalake_requests( f'{request_url}/comments', 'get', self._get_headers(), None) final_dict['comments'] = response_dict dict_threat['results'].append(final_dict) return dict_threat, list_of_lost_hashes
def pretty_print(raw_response, stdout_format): """ takes the API raw response and format it for be printed as stdout stdout_format possible values {json, csv} """ if stdout_format == 'json': logger.info(json.dumps(raw_response, indent=4, sort_keys=True)) return if stdout_format == 'csv': logger.info(raw_response) return blue_bg = '\033[104m' eol = '\x1b[0m' boolean_to_text_and_color = { True: ('FOUND', '\x1b[6;30;42m'), False: ('NOT_FOUND', '\x1b[6;30;41m') } for atom_type in raw_response.keys(): logger.info( f'{blue_bg}{"#" * 60} {atom_type.upper()} {"#" * (60 - len(atom_type))}{eol}' ) for atom in raw_response[atom_type]: found = atom.get('threat_found', False) text, color = boolean_to_text_and_color[found] logger.info( f'{atom_type} {atom["atom_value"]} hashkey: {atom["hashkey"]} {color} {text} {eol}' ) logger.info('')
def defang_threats(threats, atom_type): defanged = [] # matches urls like http://www.website.com:444/file.html standard_url_regex = re.compile(r'^(https?:\/\/)[a-z0-9]+([\-\.][a-z0-9]+)*\.[a-z]{2,5}(:[0-9]{1,5})?(\/.*)?$') # matches urls like http://185.25.5.3:8080/result.php (ipv4 or ipv6) ip_url_regex = re.compile(r'^(https?:\/\/)[0-9a-zA-Z]{1,4}([\.:][0-9a-zA-Z]{1,4}){3,7}(:[0-9]{1,5})?(\/.*)?$') for threat in threats: unmodified_threat = threat threat = threat.replace('[.]', '.') threat = threat.replace('(.)', '.') if atom_type == 'url': if not threat.startswith('http'): if threat.startswith('hxxp'): threat = threat.replace('hxxp', 'http') elif threat.startswith('ftp'): threat = threat.replace('ftp', 'http') elif threat.startswith('sftp'): threat = threat.replace('sftp', 'https') else: threat = 'http://' + threat if not standard_url_regex.match(threat) and not ip_url_regex.match(threat): logger.warning(f'\'{unmodified_threat}\' has been modified as \'{threat}\' but is still not recognized' f' as an url. Skipping this line') continue if unmodified_threat != threat: logger.info(f'\'{unmodified_threat}\' has been modified as \'{threat}\'') defanged.append(threat) return defanged
def lookup_threats(self, threats: list, atom_type, hashkey_only): boolean_to_text_and_color = {True: ('FOUND', '\x1b[6;30;42m'), False: ('NOT_FOUND', '\x1b[6;30;41m')} complete_response = None for threat in threats: response = self.get_lookup_result(threat, atom_type, hashkey_only) if not response: continue found = response['threat_found'] if 'threat_found' in response.keys() else True text, color = boolean_to_text_and_color[found] logger.info('{}{} hashkey:{} {}\x1b[0m'.format(color, threat, response['hashkey'], text)) complete_response = {} if not complete_response else complete_response complete_response[threat] = response return complete_response
def main(override_args=None): """Method to start the script""" starter = BaseScripts() logger.debug(f'START: get_threats_from_query_hash.py') # Load initial args parser = starter.start( 'Retrieve a list of response from a given query hash.') parser.add_argument( '--query_fields', help= 'fields to be retrieved from the threat (default: only the hashkey)\n' 'If an atom detail isn\'t present in a particular atom, empty string is returned.', nargs='+', default=['threat_hashkey'], ) parser.add_argument( '--list', help= 'Turn the output in a list (require query_fields to be a single element)', action='store_true', ) required_named = parser.add_argument_group('required arguments') required_named.add_argument( 'query_hash', help='the query hash from which to retrieve the response hashkeys', ) if override_args: args = parser.parse_args(override_args) else: args = parser.parse_args() if len(args.query_fields) > 1 and args.list: parser.error( "List output format is only available if a single element is queried (via query_fields)" ) # Load api_endpoints and tokens endpoint_config, main_url, tokens = starter.load_config(args) logger.debug( f'Start to search for threat from the query hash:{args.query_hash}') bulk_search = BulkSearch(endpoint_config, args.env, tokens) response = bulk_search.get_threats(args.query_hash, args.query_fields) original_count = response.get('count', 0) logger.info(f'Number of threat that have been retrieved: {original_count}') formatted_output = format_output(response, args.list) if args.output: with open(args.output, 'w') as output: output.write(formatted_output) else: logger.info(formatted_output) if args.output: logger.info(f'Threats saved in {args.output}') else: logger.info('Done')
def post_new_score_from_list(self, hashkeys: list, scores: Dict[str, int], override_type: str = 'temporary') -> list: """ Post new score to the API from a list of hashkeys """ return_value = [] for hashkey in hashkeys: response = self._post_new_score(hashkey, scores, override_type) if response.get('message'): logger.warning('\x1b[6;30;41m' + hashkey + ': ' + response.get('message') + '\x1b[0m') return_value.append(hashkey + ': ' + response.get('message')) else: return_value.append(hashkey + ': OK') logger.info('\x1b[6;30;42m' + hashkey + ': OK\x1b[0m') return return_value
def post_tags(self, hashkeys: Set[str], tags: List[str], *, public=True) -> list: """ Post tags on threat hashkeys """ visibility = 'public' if public else 'organization' return_value = [] for hashkey in hashkeys: response = self._post_tags_to_hashkey(hashkey, tags, visibility) if response.get('message'): logger.warning('\x1b[6;30;41m' + hashkey + ': ' + response.get('message') + '\x1b[0m') return_value.append(hashkey + ': ' + response.get('message')) else: return_value.append(hashkey + ': OK') logger.info('\x1b[6;30;42m' + hashkey + ': OK\x1b[0m') return return_value
def post_comments_and_tags_from_list(self, hashkeys: Set[str], content: str, tags: list, *, public=True) -> list: """ Post comments and tag on threats hashkey """ visibility = 'public' if public else 'organization' return_value = [] for hashkey in hashkeys: response = self._post_comments_and_tags(hashkey, content, tags, visibility) if response.get('message'): logger.warning('\x1b[6;30;41m' + hashkey + ': ' + response.get('message') + '\x1b[0m') return_value.append(hashkey + ': ' + response.get('message')) else: return_value.append(hashkey + ': OK') logger.info('\x1b[6;30;42m' + hashkey + ': OK\x1b[0m') return return_value
def add_threats(self, atom_list: list, atom_type: str, is_whitelist: bool, threats_score: Dict[str, int], is_public: bool, tags: list, links: list, override_type: str) -> dict: """ Use it to add a list of threats to the API. :param atom_list: atoms that needs to be added. :param atom_type: must be one of the _authorized_atom_value :param is_whitelist: if true the score will be set to 0 :param threats_score: a dict that contain {threat_type -> score} :param is_public: if true the added threat will be public else will be reserved to organization :param tags: a list of tags to add :param links: external_analysis_link to include with each atoms :param override_type: either 'permanent' or 'temporary'. Permanent don't allow future automatic score change """ payload = { 'override_type': override_type, 'public': is_public, 'threat_data': { 'content': {}, 'scores': [], 'threat_types': [], 'tags': tags } } if is_whitelist: for threat in self.authorized_threats_value: payload['threat_data']['scores'].append({'score': {'risk': 0}, 'threat_type': threat}) payload['threat_data']['threat_types'].append(threat) else: for threat, score in threats_score.items(): payload['threat_data']['scores'].append({'score': {'risk': score}, 'threat_type': threat}) payload['threat_data']['threat_types'].append(threat) return_value = {'results': []} for atom in atom_list: if not atom: # empty value logger.info(f'EMPTY ATOM {atom.ljust(self.terminal_size - 6, " ")} \x1b[0;30;41m KO \x1b[0m') continue response_dict = self._add_new_atom(atom, atom_type, payload, links) if response_dict.get('atom_value'): logger.info(atom.ljust(self.terminal_size - 6, ' ') + '\x1b[0;30;42m' + ' OK ' + '\x1b[0m') return_value['results'].append(response_dict) else: logger.info(atom.ljust(self.terminal_size - 6, ' ') + '\x1b[0;30;41m' + ' KO ' + '\x1b[0m') logger.debug(response_dict) return return_value
def main(override_args=None): """Method to start the script""" starter = BaseScripts() # Load initial args parser = starter.start('Edit scores of a specified list of ids (hashkeys)') parser.add_argument( 'hashkeys', help='hashkeys of the threat to edit score.', nargs='*', ) parser.add_argument( '-i', '--input_file', help='hashkey txt file, with one hashkey by line.', ) parser.add_argument( '-t', '--threat_types', nargs='+', help= 'Choose specific threat types and their score, like: ddos 50 scam 15.', ) parser.add_argument( '--permanent', help= '''Permanent: all values will override any values provided by both newer and older IOCs. Newer IOCs with override_type permanent can still override old permanent changes. temporary: all values should override any values provided by older IOCs, but not newer ones.''', action='store_true', ) if override_args: args = parser.parse_args(override_args) else: args = parser.parse_args() logger.debug(f'START: edit_score.py') if not args.hashkeys and not args.input_file: parser.error("either a hashkey or an input_file is required") if not args.threat_types or len(args.threat_types) % 2 != 0: parser.error("threat_types invalid ! should be like: ddos 50 scam 15") parsed_threat_type = AddThreatsPost.parse_threat_types(args.threat_types) hashkeys = set(args.hashkeys) if args.hashkeys else set() if args.input_file: retrieve_hashkeys_from_file(args.input_file, hashkeys) # Load api_endpoints and tokens endpoint_url, main_url, tokens = starter.load_config(args) url_threats = main_url + endpoint_url['endpoints']['threats'] post_engine_edit_score = ThreatsScoringPost(url_threats, main_url, tokens) response_dict = post_engine_edit_score.post_new_score_from_list( hashkeys, parsed_threat_type, 'permanent' if args.permanent else 'temporary', ) if args.output: starter.save_output(args.output, response_dict) logger.info(f'Results saved in {args.output}\n') logger.debug(f'END: edit_score.py')