def queue_bulk_threats(self, atom_list, payload):
hashkey_created = []
bulk_in_flight = [] # bulk task uuid unchecked
for batch in split_list(atom_list, self._batch_size()):
if len(bulk_in_flight) >= self.OCD_DTL_MAX_BULK_THREATS_IN_FLIGHT:
bulk_threat_task_uuid = bulk_in_flight.pop(0)
hashkey_created += self.check_bulk_threats_added(bulk_threat_task_uuid)
payload['atom_values'] = '\n'.join(batch) # Raw csv expected
response = self.datalake_requests(self.url, 'post', self._post_headers(), payload)
task_uid = response.get('task_uuid')
if task_uid:
bulk_in_flight.append(response['task_uuid'])
else:
logger.warning(f'batch of threats from {batch[0]} to {batch[-1]} failed to be created')
# Finish to check the other bulk tasks
for bulk_threat_task_uuid in bulk_in_flight:
hashkey_created += self.check_bulk_threats_added(bulk_threat_task_uuid)
nb_threats = len(hashkey_created)
if nb_threats > 0:
ok_sign = '\x1b[0;30;42m' + ' OK ' + '\x1b[0m'
logger.info(f'Created {nb_threats} threats'.ljust(self.terminal_size - 6, ' ') + ok_sign)
else:
ko_sign = '\x1b[0;30;41m' + ' KO ' + '\x1b[0m'
logger.info(f'Failed to create any threats'.ljust(self.terminal_size - 6, ' ') + ko_sign)
return set(hashkey_created)
def main(override_args=None):
starter = BaseScripts()
logger.debug(f'START: get_query_hash.py')
# Load initial args
parser = starter.start('Retrieve a query hash from a query body (a json used for the Advanced Search).')
required_named = parser.add_argument_group('required arguments')
required_named.add_argument(
'query_body_path',
help='path to the json file containing the query body',
)
if override_args:
args = parser.parse_args(override_args)
else:
args = parser.parse_args()
# Load api_endpoints and tokens
endpoint_config, main_url, tokens = starter.load_config(args)
with open(args.query_body_path, 'r') as query_body_file:
query_body = json.load(query_body_file)
logger.debug(f'Retrieving query hash for query body: {query_body}')
advanced_search = AdvancedSearch(endpoint_config, args.env, tokens)
response = advanced_search.get_threats(query_body, limit=0)
if not response or 'query_hash' not in response:
logger.error("Couldn't retrieve a query hash, is the query body valid ?")
exit(1)
query_hash = response['query_hash']
if args.output:
with open(args.output, 'w') as output:
output.write(query_hash)
logger.info(f'Query hash saved in {args.output}')
else:
logger.info(f'Query hash associated: {query_hash}')
def get_json(self, list_threats: list):
"""
Retrieve the JSON file of a list of threats and their comments.
:return dict, list: threats found and a list of hashkey not found
"""
total_hash = len(list_threats)
dict_threat = {'count': total_hash, 'results': []}
list_of_lost_hashes = []
for index, threat in enumerate(list_threats):
request_url = self.url + threat
response_dict = self.datalake_requests(
request_url,
'get',
self._get_headers(),
None,
)
final_dict = response_dict
if not response_dict.get('hashkey'):
list_of_lost_hashes.append(threat)
logger.info(
f'{str(index).ljust(5)}:{threat.ljust(self.terminal_size - 11)}\x1b[0;30;41mERROR\x1b[0m'
)
else:
logger.info(
f'{str(index).ljust(5)}:{threat.ljust(self.terminal_size - 10)}\x1b[0;30;42m OK \x1b[0m'
)
response_dict = self.datalake_requests(
f'{request_url}/comments', 'get', self._get_headers(),
None)
final_dict['comments'] = response_dict
dict_threat['results'].append(final_dict)
return dict_threat, list_of_lost_hashes
def pretty_print(raw_response, stdout_format):
"""
takes the API raw response and format it for be printed as stdout
stdout_format possible values {json, csv}
"""
if stdout_format == 'json':
logger.info(json.dumps(raw_response, indent=4, sort_keys=True))
return
if stdout_format == 'csv':
logger.info(raw_response)
return
blue_bg = '\033[104m'
eol = '\x1b[0m'
boolean_to_text_and_color = {
True: ('FOUND', '\x1b[6;30;42m'),
False: ('NOT_FOUND', '\x1b[6;30;41m')
}
for atom_type in raw_response.keys():
logger.info(
f'{blue_bg}{"#" * 60} {atom_type.upper()} {"#" * (60 - len(atom_type))}{eol}'
)
for atom in raw_response[atom_type]:
found = atom.get('threat_found', False)
text, color = boolean_to_text_and_color[found]
logger.info(
f'{atom_type} {atom["atom_value"]} hashkey: {atom["hashkey"]} {color} {text} {eol}'
)
logger.info('')
def defang_threats(threats, atom_type):
defanged = []
# matches urls like http://www.website.com:444/file.html
standard_url_regex = re.compile(r'^(https?:\/\/)[a-z0-9]+([\-\.][a-z0-9]+)*\.[a-z]{2,5}(:[0-9]{1,5})?(\/.*)?$')
# matches urls like http://185.25.5.3:8080/result.php (ipv4 or ipv6)
ip_url_regex = re.compile(r'^(https?:\/\/)[0-9a-zA-Z]{1,4}([\.:][0-9a-zA-Z]{1,4}){3,7}(:[0-9]{1,5})?(\/.*)?$')
for threat in threats:
unmodified_threat = threat
threat = threat.replace('[.]', '.')
threat = threat.replace('(.)', '.')
if atom_type == 'url':
if not threat.startswith('http'):
if threat.startswith('hxxp'):
threat = threat.replace('hxxp', 'http')
elif threat.startswith('ftp'):
threat = threat.replace('ftp', 'http')
elif threat.startswith('sftp'):
threat = threat.replace('sftp', 'https')
else:
threat = 'http://' + threat
if not standard_url_regex.match(threat) and not ip_url_regex.match(threat):
logger.warning(f'\'{unmodified_threat}\' has been modified as \'{threat}\' but is still not recognized'
f' as an url. Skipping this line')
continue
if unmodified_threat != threat:
logger.info(f'\'{unmodified_threat}\' has been modified as \'{threat}\'')
defanged.append(threat)
return defanged
def lookup_threats(self, threats: list, atom_type, hashkey_only):
boolean_to_text_and_color = {True: ('FOUND', '\x1b[6;30;42m'),
False: ('NOT_FOUND', '\x1b[6;30;41m')}
complete_response = None
for threat in threats:
response = self.get_lookup_result(threat, atom_type, hashkey_only)
if not response:
continue
found = response['threat_found'] if 'threat_found' in response.keys() else True
text, color = boolean_to_text_and_color[found]
logger.info('{}{} hashkey:{} {}\x1b[0m'.format(color, threat, response['hashkey'], text))
complete_response = {} if not complete_response else complete_response
complete_response[threat] = response
return complete_response
def main(override_args=None):
"""Method to start the script"""
starter = BaseScripts()
logger.debug(f'START: get_threats_from_query_hash.py')
# Load initial args
parser = starter.start(
'Retrieve a list of response from a given query hash.')
parser.add_argument(
'--query_fields',
help=
'fields to be retrieved from the threat (default: only the hashkey)\n'
'If an atom detail isn\'t present in a particular atom, empty string is returned.',
nargs='+',
default=['threat_hashkey'],
)
parser.add_argument(
'--list',
help=
'Turn the output in a list (require query_fields to be a single element)',
action='store_true',
)
required_named = parser.add_argument_group('required arguments')
required_named.add_argument(
'query_hash',
help='the query hash from which to retrieve the response hashkeys',
)
if override_args:
args = parser.parse_args(override_args)
else:
args = parser.parse_args()
if len(args.query_fields) > 1 and args.list:
parser.error(
"List output format is only available if a single element is queried (via query_fields)"
)
# Load api_endpoints and tokens
endpoint_config, main_url, tokens = starter.load_config(args)
logger.debug(
f'Start to search for threat from the query hash:{args.query_hash}')
bulk_search = BulkSearch(endpoint_config, args.env, tokens)
response = bulk_search.get_threats(args.query_hash, args.query_fields)
original_count = response.get('count', 0)
logger.info(f'Number of threat that have been retrieved: {original_count}')
formatted_output = format_output(response, args.list)
if args.output:
with open(args.output, 'w') as output:
output.write(formatted_output)
else:
logger.info(formatted_output)
if args.output:
logger.info(f'Threats saved in {args.output}')
else:
logger.info('Done')
def post_new_score_from_list(self, hashkeys: list, scores: Dict[str, int],
override_type: str = 'temporary') -> list:
"""
Post new score to the API from a list of hashkeys
"""
return_value = []
for hashkey in hashkeys:
response = self._post_new_score(hashkey, scores, override_type)
if response.get('message'):
logger.warning('\x1b[6;30;41m' + hashkey + ': ' + response.get('message') + '\x1b[0m')
return_value.append(hashkey + ': ' + response.get('message'))
else:
return_value.append(hashkey + ': OK')
logger.info('\x1b[6;30;42m' + hashkey + ': OK\x1b[0m')
return return_value
def post_tags(self, hashkeys: Set[str], tags: List[str], *, public=True) -> list:
"""
Post tags on threat hashkeys
"""
visibility = 'public' if public else 'organization'
return_value = []
for hashkey in hashkeys:
response = self._post_tags_to_hashkey(hashkey, tags, visibility)
if response.get('message'):
logger.warning('\x1b[6;30;41m' + hashkey + ': ' + response.get('message') + '\x1b[0m')
return_value.append(hashkey + ': ' + response.get('message'))
else:
return_value.append(hashkey + ': OK')
logger.info('\x1b[6;30;42m' + hashkey + ': OK\x1b[0m')
return return_value
def post_comments_and_tags_from_list(self,
hashkeys: Set[str],
content: str,
tags: list,
*,
public=True) -> list:
"""
Post comments and tag on threats hashkey
"""
visibility = 'public' if public else 'organization'
return_value = []
for hashkey in hashkeys:
response = self._post_comments_and_tags(hashkey, content, tags,
visibility)
if response.get('message'):
logger.warning('\x1b[6;30;41m' + hashkey + ': ' +
response.get('message') + '\x1b[0m')
return_value.append(hashkey + ': ' + response.get('message'))
else:
return_value.append(hashkey + ': OK')
logger.info('\x1b[6;30;42m' + hashkey + ': OK\x1b[0m')
return return_value
def add_threats(self, atom_list: list, atom_type: str, is_whitelist: bool, threats_score: Dict[str, int],
is_public: bool, tags: list, links: list, override_type: str) -> dict:
"""
Use it to add a list of threats to the API.
:param atom_list: atoms that needs to be added.
:param atom_type: must be one of the _authorized_atom_value
:param is_whitelist: if true the score will be set to 0
:param threats_score: a dict that contain {threat_type -> score}
:param is_public: if true the added threat will be public else will be reserved to organization
:param tags: a list of tags to add
:param links: external_analysis_link to include with each atoms
:param override_type: either 'permanent' or 'temporary'. Permanent don't allow future automatic score change
"""
payload = {
'override_type': override_type,
'public': is_public,
'threat_data': {
'content': {},
'scores': [],
'threat_types': [],
'tags': tags
}
}
if is_whitelist:
for threat in self.authorized_threats_value:
payload['threat_data']['scores'].append({'score': {'risk': 0}, 'threat_type': threat})
payload['threat_data']['threat_types'].append(threat)
else:
for threat, score in threats_score.items():
payload['threat_data']['scores'].append({'score': {'risk': score}, 'threat_type': threat})
payload['threat_data']['threat_types'].append(threat)
return_value = {'results': []}
for atom in atom_list:
if not atom: # empty value
logger.info(f'EMPTY ATOM {atom.ljust(self.terminal_size - 6, " ")} \x1b[0;30;41m KO \x1b[0m')
continue
response_dict = self._add_new_atom(atom, atom_type, payload, links)
if response_dict.get('atom_value'):
logger.info(atom.ljust(self.terminal_size - 6, ' ') + '\x1b[0;30;42m' + ' OK ' + '\x1b[0m')
return_value['results'].append(response_dict)
else:
logger.info(atom.ljust(self.terminal_size - 6, ' ') + '\x1b[0;30;41m' + ' KO ' + '\x1b[0m')
logger.debug(response_dict)
return return_value
def main(override_args=None):
"""Method to start the script"""
starter = BaseScripts()
# Load initial args
parser = starter.start('Edit scores of a specified list of ids (hashkeys)')
parser.add_argument(
'hashkeys',
help='hashkeys of the threat to edit score.',
nargs='*',
)
parser.add_argument(
'-i',
'--input_file',
help='hashkey txt file, with one hashkey by line.',
)
parser.add_argument(
'-t',
'--threat_types',
nargs='+',
help=
'Choose specific threat types and their score, like: ddos 50 scam 15.',
)
parser.add_argument(
'--permanent',
help=
'''Permanent: all values will override any values provided by both newer and
older IOCs. Newer IOCs with override_type permanent can still override old permanent changes.
temporary: all values should override any values provided by older IOCs,
but not newer ones.''',
action='store_true',
)
if override_args:
args = parser.parse_args(override_args)
else:
args = parser.parse_args()
logger.debug(f'START: edit_score.py')
if not args.hashkeys and not args.input_file:
parser.error("either a hashkey or an input_file is required")
if not args.threat_types or len(args.threat_types) % 2 != 0:
parser.error("threat_types invalid ! should be like: ddos 50 scam 15")
parsed_threat_type = AddThreatsPost.parse_threat_types(args.threat_types)
hashkeys = set(args.hashkeys) if args.hashkeys else set()
if args.input_file:
retrieve_hashkeys_from_file(args.input_file, hashkeys)
# Load api_endpoints and tokens
endpoint_url, main_url, tokens = starter.load_config(args)
url_threats = main_url + endpoint_url['endpoints']['threats']
post_engine_edit_score = ThreatsScoringPost(url_threats, main_url, tokens)
response_dict = post_engine_edit_score.post_new_score_from_list(
hashkeys,
parsed_threat_type,
'permanent' if args.permanent else 'temporary',
)
if args.output:
starter.save_output(args.output, response_dict)
logger.info(f'Results saved in {args.output}\n')
logger.debug(f'END: edit_score.py')
