def send_query(params, url):
try:
conn = httplib.HTTPSConnection(
params['server'], context=ssl._create_unverified_context())
conn.request("GET", url, None, headers={'X-Auth-Token': params['key']})
return conn.getresponse()
except Exception as err:
jfutil.debug(True, "Query \'{}\' failed\n{}".format(url, str(err)))
exit()
def check_params(params):
# IF JETFREQ IS RUN IN AN 'EVENT' MODE, CHECK THERE IS EXACTLY ONE EVENT TYPE FLAG (EXCLUDING CROSSPROCS)
if params['mode'] == 'BY_EVENT' or params['mode'] == 'COMPARE_EVENT':
flags = jfutil.get_event_type_flags(params)
if len(flags) > 1:
raise jfexceptions.IncorrectFlagUsageForModeError(
params['mode'], flags)
if len(flags) == 0:
raise jfexceptions.ByEventModeFlagRequiredError(params['mode'])
if 'x' in flags:
raise jfexceptions.FlagsNotApplicableError(params['mode'], 'x')
# IF JETFREQ IS RUN IN A 'PROCESS' MODE, CHECK THERE IS ONE OR MORE EVENT TYPE FLAGS
if params['mode'] == 'BY_PROCESS' or params['mode'] == 'COMPARE_PROCESS':
if params['regmods'] == False and params[
'filemods'] == False and params[
'childprocs'] == False and params[
'netconns'] == False and params[
'crossprocs'] == False and params[
'modloads'] == False:
raise jfexceptions.ProcessModeMissingEventTypeError(params['mode'])
# IF JETFREQ IS RUN IN A 'COMPARE' MODE, CHECK THAT A SAMPLE FILE HAS BEEN IDENTIFIED
if params['import_sample'] == None and (params['mode'] == 'COMPARE_PROCESS'
or params['mode']
== 'COMPARE_EVENT'):
raise jfexceptions.CompareModeMissingSampleFileError(params['mode'])
# IF JETFREQ IS NOT RUN IN A 'COMPARE' MODE, CHECK THAT A SAMPLE FILE HAS NOT BEEN IDENTIFIED
if re.match(
r'COMPARE_',
params['mode']) == None and not params['import_sample'] == None:
raise jfexceptions.FlagsNotApplicableError(params['mode'], 'i')
# CHECK THAT A USER NAME HAS BEEN INCLUDED OR EXCLUDED, BUT NOT BOTH
if not params['user_name'] == None and not params['exclude_user'] == None:
params['exclude_user'] == None
# CHECK THAT A HOST NAME HAS BEEN INCLUDED OR EXCLUDED, BUT NOT BOTH
if not params['host_name'] == None and not params['exclude_host'] == None:
params['exclude_host'] == None
# CHECK THAT THE GREATER-THAN AND LESS-THAN THRESHOLD VALUES ARE NOT EQUAL
if params['threshold_gt'] == params['threshold_lt']:
raise jfexceptions.NonsensicalThresholdValuesError(
params['threshold_gt'], params['threshold_lt'])
# DO NOT TRUNCATE IF WRITING
if params['write_file'] and params['truncate']:
params['truncate'] = False
jfutil.debug(
True,
'JetFreq does not truncate when writing to file. Ignoring -k flag.'
)
return params
def compare_event(params, representative_sample, target_sample):
jfutil.debug(
params['verbose'],
'Comparing target event {} to representative sample {}'.format(
params['search_name'], params['import_sample']))
# INITIALIZE THE JSON CONTAINER FOR PROCESS DIFFERENCES
event_diffs = []
difftype = DiffType()
rep_dict = load_into_dict(representative_sample)
tar_dict = load_into_dict(target_sample)
# FOR EACH PROCESS IN THE TAR SAMPLE
for key in tar_dict:
# IF THE PROCESS DOES NOT EXIST IN THE REP SAMPLE, ADD A 'MISSING FROM REP' DIFF TYPE
if not key in rep_dict:
event_diffs.append(
EventDiff(tar_dict[key]['object'], None, difftype.MISS_FM_REP))
else:
# IF THE EVENT EXISTS IN BOTH SAMPLES, CALCULATE WHETHER IT OCCURS MORE IN THE REP OR TAR SAMPLES
if float(tar_dict[key]['freq']) > float(rep_dict[key]['freq']):
event_diffs.append(
EventDiff(tar_dict[key]['object'], rep_dict[key]['object'],
difftype.HIGH_FQ_TAR))
elif float(tar_dict[key]['freq']) < float(rep_dict[key]['freq']):
event_diffs.append(
EventDiff(tar_dict[key]['object'], rep_dict[key]['object'],
difftype.HIGH_FQ_REP))
# FOR EACH PROCESS IN THE REP SAMPLE
for key in rep_dict:
# IF THE PROCESS DOES NOT EXIST IN THE TAR SAMPLE, ADD A 'MISSING FROM TAR' DIFF TYPE
if not key in tar_dict:
event_diffs.append(
EventDiff(None, rep_dict[key]['object'], difftype.MISS_FM_TAR))
jfutil.debug(params['verbose'], 'Comparison complete')
return event_diffs
def format_query(params):
query = ""
# IF THE MODE IS AN 'EVENT' MODE, ONLY ONE EVENT TYPE IS ADDED TO THE QUERY
if params['mode'] == 'BY_EVENT' or params['mode'] == 'COMPARE_EVENT':
if params['modloads'] == True:
query += "modload%3A{}".format(params['search_name'])
elif params['regmods'] == True:
query += "regmod%3A{}".format(params['search_name'])
elif params['childprocs'] == True:
query += "childproc_name%3A{}".format(params['search_name'])
elif params['filemods'] == True:
query += "filemod%3A{}".format(params['search_name'])
elif params['netconns'] == True:
query += "domain%3A{}".format(params['search_name'])
query += "%20start%3A{}".format(params['start_time'])
else:
# IF THE MODE IS A 'PROCESS' MODE, ADD THE PROCESS SEARCH NAME ONLY
query += "process_name%3A{}%20start%3A{}".format(
params['search_name'], params['start_time'])
# IF A USER OR EXCLUDE-USER HAS BEEN INCLUDED, APPEND IT TO THE QUERY
if params['user_name'] != None:
query += "%20username%3A{}".format(params['user_name'])
elif params['exclude_user'] != None:
query += "%20-username%3A{}".format(params['exclude_user'])
# IF A HOST OR EXLUDE-HOST HAS BEEN INCLUDED, APPEND IT TO THE QUERY
if params['host_name'] != None:
query += "%20hostname%3A{}".format(params['host_name'])
elif params['exclude_host'] != None:
query += "%20-hostname%3A{}".format(params['exclude_host'])
jfutil.debug(params['verbose'], "Query formatted \'{}\'".format(query))
return query
def analyze_by_event(params, data):
jfutil.debug(params['verbose'],
"Conducting analysis for {}".format(params['search_name']))
# COUNT HOW MANY PROCESSES CREATE A GIVEN EVENT IN THE SAMPLE
event_counts = {}
for process in data:
if not process['path'] in event_counts:
event_counts[process['path']] = 1
else:
event_counts[process['path']] = event_counts[process['path']] + 1
jfutil.debug(params['verbose'], "Count complete")
# CALCULATE THE FREQUENCY IN WHICH PROCESSES CREATE THE EVENT
event_freqs = calculate_event_frequency(event_counts, len(data),
params['threshold_gt'],
params['threshold_lt'])
jfutil.debug(params['verbose'], "Freq calculations complete")
return event_freqs
# MAIN METHOD FOR JETFREQ.PY
try:
# IMPORT THE ARGUMENTS FROM COMMAND LINE AND LOAD
# AND LOAD THEM INTO A JSON OF PARAMETERS
# THAT CAN BE EASILY PASSED BETWEEN FUNCTIONS
if __name__ == "__main__":
params = jfparser.process_params(sys.argv)
# CHECK IF JETFREQ IS RUNNING IN HELP MODE
# IF SO, SHOW HELP AND EXIT
if params['mode'] == 'SHOW_HELP':
jfutil.show_help()
exit()
# DEBUG INFO
jfutil.debug(params['verbose'], "Running in {} mode".format(params['mode']))
jfutil.debug(params['verbose'], 'Parameters parsed as {}'.format(params))
# SHOW THE JETFREQ BANNER
banner = [" _ __ ___ "," (_)__ / /_/ _/______ ___ _",
" / / -_) __/ _/ __/ -_) _ `/"," __/ /\__/\__/_//_/ \__/\_, /",
"|___/ /_/"]
jfutil.debug(True, banner)
# CHECK FOR ATTEMPTS TO GENERATE LARGE SAMPLES AND WARN USER
jfutil.throttle(params)
# GET DATA FOR EVENT OR PROCESS FROM CBR SERVER AND ANALYZE IT
representative_sample = None
target_sample = None
data = None
def compare_process(params, representative_sample, target_sample):
jfutil.debug(
params['verbose'],
'Comparing target process {} to representative sample {}'.format(
params['search_name'], params['import_sample']))
# INITIALIZE THE JSON CONTAINER FOR EVENT DIFFERENCES
event_diffs = {
'modloads': [],
'regmods': [],
'childprocs': [],
'filemods': [],
'netconns': [],
'crossprocs': []
}
difftype = DiffType()
# FOR EACH EVENT TYPE
for key in event_diffs:
# IF THE EVENT TYPE EXISTS IN THE REP SAMPLE AND DOES NOT EXIST IN THE TAR SAMPLE
if not representative_sample[key] == None and target_sample[
key] == None:
# FOR ALL EVENTS OF THIS TYPE IN THE REP SAMPLE, ADD AN APPROPRIATE 'MISSING FROM TAR' DIFF TYPE
for event in representative_sample[key]:
event_diffs[key].append(
EventDiff(None, event,
difftype.get_diff_type_by_event(key, 'rep')))
# IF THE EVENT TYPE EXISTS IN THE TAR SAMPLE AND DOES NOT EXIST IN THE REP SAMPLE
elif not target_sample[key] == None and representative_sample[
key] == None:
# FOR ALL EVENTS OF THIS TYPE IN THE TAR SAMPLE, ADD AN APPROPRIATE 'MISSING FROM REP' DIFF TYPE
for event in target_sample[key]:
event_diffs[key].append(
EventDiff(None, event,
difftype.get_diff_type_by_event(key, 'tar')))
# IF THE EVENT TYPE EXISTS IN BOTH THE REP AND TAR SAMPLES
elif not target_sample[key] == None and not representative_sample[
key] == None:
# LOAD THE EVENT LIST INTO A DICTIONARY
rep_dict = load_into_dict(representative_sample[key])
tar_dict = load_into_dict(target_sample[key])
# FOR EACH EVENT OF THIS TYPE IN THE TAR SAMPLE
for sub_key in tar_dict:
# IF THE EVENT DOES NOT EXIST IN THE REP SAMPLE, ADD A 'MISSING FROM REP' DIFF TYPE
if not sub_key in rep_dict:
event_diffs[key].append(
EventDiff(tar_dict[sub_key]['object'], None,
difftype.MISS_FM_REP))
else:
# IF THE EVENT EXISTS IN BOTH SAMPLES, CALCULATE WHETHER IT OCCURS MORE IN THE REP OR TAR SAMPLES
if float(tar_dict[sub_key]['freq']) > float(
rep_dict[sub_key]['freq']):
event_diffs[key].append(
EventDiff(tar_dict[sub_key]['object'],
rep_dict[sub_key]['object'],
difftype.HIGH_FQ_TAR))
elif float(tar_dict[sub_key]['freq']) < float(
rep_dict[sub_key]['freq']):
event_diffs[key].append(
EventDiff(tar_dict[sub_key]['object'],
rep_dict[sub_key]['object'],
difftype.HIGH_FQ_REP))
# FOR EACH EVENT OF THIS TYPE IN THE REP SAMPLE
for sub_key in rep_dict:
# IF THE EVENT DOES NOT EXIST IN THE TAR SAMPLE, ADD A 'MISSING FROM TAR' DIFF TYPE
if not sub_key in tar_dict:
event_diffs[key].append(
EventDiff(None, rep_dict[sub_key]['object'],
difftype.MISS_FM_TAR))
jfutil.debug(params['verbose'], 'Comparison complete')
return event_diffs
def analyze_by_process(params, data):
jfutil.debug(params['verbose'],
"Conducting analysis for {}".format(params['search_name']))
# INITIALIZE THE JSON CONTAINER FOR THE RESULTS OF THE EVENT COUNT
event_counts = {
'modloads': {},
'regmods': {},
'childprocs': {},
'filemods': {},
'netconns': {},
'crossprocs': {}
}
# COUNT HOW MANY TIMES EACH SPECIFIC EVENT OCCURS ACROSS ALL
# PROCESSES IN THE SAMPLE RETURNED FROM CARBON BLACK
for process in data:
if 'modloads' in process:
jfutil.debug(params['verbose'], 'Counting modloads')
event_counts['modloads'] = count_events(process,
event_counts['modloads'],
'modloads')
if 'regmods' in process:
jfutil.debug(params['verbose'], 'Counting regmods')
event_counts['regmods'] = count_events(process,
event_counts['regmods'],
'regmods')
if 'childprocs' in process:
jfutil.debug(params['verbose'], 'Counting childprocs')
event_counts['childprocs'] = count_events(
process, event_counts['childprocs'], 'childprocs')
if 'filemods' in process:
jfutil.debug(params['verbose'], 'Counting filemods')
event_counts['filemods'] = count_events(process,
event_counts['filemods'],
'filemods')
if 'netconns' in process:
jfutil.debug(params['verbose'], 'Counting netconns')
event_counts['netconns'] = count_events(process,
event_counts['netconns'],
'netconns')
if 'crossprocs' in process:
jfutil.debug(params['verbose'], 'Counting crossprocs')
event_counts['crossprocs'] = count_events(
process, event_counts['crossprocs'], 'crossprocs')
jfutil.debug(params['verbose'], 'Count complete')
# INITIALIZE THE JSON CONTAINER FOR THE RESULTS OF THE EVENT FREQUENCY ANALYSIS
event_freqs = {
'modloads': None,
'regmods': None,
'childprocs': None,
'filemods': None,
'netconns': None,
'crossprocs': None
}
# USING THE COUNTS, CALCULATE THE FREQUENCY IN WHICH EACH EVENT OCCURS ACROSS
# PROCESSES IN THE SAMPLE RETURNED BY CARBON BLACK
if len(event_counts['modloads']) > 0:
jfutil.debug(params['verbose'], 'Calculating freq of modloads')
event_freqs['modloads'] = calculate_event_frequency(
event_counts['modloads'], len(data), params['threshold_gt'],
params['threshold_lt'])
if len(event_counts['regmods']) > 0:
jfutil.debug(params['verbose'], 'Calculating freq of regmods')
event_freqs['regmods'] = calculate_event_frequency(
event_counts['regmods'], len(data), params['threshold_gt'],
params['threshold_lt'])
if len(event_counts['childprocs']) > 0:
jfutil.debug(params['verbose'], 'Calculating freq of childprocs')
event_freqs['childprocs'] = calculate_event_frequency(
event_counts['childprocs'], len(data), params['threshold_gt'],
params['threshold_lt'])
if len(event_counts['filemods']) > 0:
jfutil.debug(params['verbose'], 'Calculating freq of filemods')
event_freqs['filemods'] = calculate_event_frequency(
event_counts['filemods'], len(data), params['threshold_gt'],
params['threshold_lt'])
if len(event_counts['netconns']) > 0:
jfutil.debug(params['verbose'], 'Calculating freq of netconns')
event_freqs['netconns'] = calculate_event_frequency(
event_counts['netconns'], len(data), params['threshold_gt'],
params['threshold_lt'])
if len(event_counts['crossprocs']) > 0:
jfutil.debug(params['verbose'], 'Calculating freq of crossprocs')
event_freqs['crossprocs'] = calculate_event_frequency(
event_counts['crossprocs'], len(data), params['threshold_gt'],
params['threshold_lt'])
jfutil.debug(params['verbose'], "Freq calculations complete")
return event_freqs
def get_data_for_event(params):
# FORMAT THE URL, INCLUDING THE QUERY
query = format_query(params)
url = "/api/v1/process?cb.urlver=1&rows={}&start=0&q={}".format(
params['sample_size'], query)
jfutil.debug(params['verbose'],
'Attempting to send query to Carbon Black server')
jfutil.debug(params['verbose'], 'Server: {}'.format(params['server']))
jfutil.debug(params['verbose'], 'API Key: {}'.format(params['key']))
jfutil.debug(params['verbose'], 'URL: {}'.format(url))
# SEND THE QUERY TO THE CARBON BLACK SERVER AND STORE THE RESULT
response = send_query(params, url)
jfutil.debug(
params['verbose'],
'Response: \'{} {}\''.format(response.status,
httplib.responses[response.status]))
# STORE THE PATH, PROCESS ID AND SEGMENT ID FOR EACH PROCESS RETURNED BY THE QUERY
data = json.loads(response.read())
process_list = []
jfutil.debug(True, 'Processing response from {}'.format(params['server']))
try:
if 'results' in data:
for result in data['results']:
process_list.append({
'path':
jfutil.homogenize_path(result['path'], 'dir',
params['homogenize']),
'process_id':
result['id'],
'segment_id':
result['segment_id']
})
except KeyError:
raise jfexceptions.UnexpectedResponseError(data)
# IF THERE ARE ANY RESULTS, RETURN THE LIST OF PROCESSES
if len(process_list) > 0:
return process_list
else:
raise jfexceptions.NoResultsError(query)
def get_data_for_process(params):
# FORMAT THE URL, INCLUDING THE QUERY
query = format_query(params)
url = "/api/v1/process?cb.urlver=1&rows={}&start=0&q={}".format(
params['sample_size'], query)
jfutil.debug(params['verbose'],
'Attempting to send query to Carbon Black server')
jfutil.debug(params['verbose'], 'Server: {}'.format(params['server']))
jfutil.debug(params['verbose'], 'API Key: {}'.format(params['key']))
jfutil.debug(params['verbose'], 'URL: {}'.format(url))
# SEND THE QUERY TO THE CARBON BLACK SERVER AND STORE THE RESULT
response = send_query(params, url)
jfutil.debug(
params['verbose'],
'Response: \'{} {}\''.format(response.status,
httplib.responses[response.status]))
# FOR EACH PROCESS RETURNED IN THE RESULTS, STORE THE PROCESS ID AND SEGMENT ID
data = json.loads(response.read())
id_list = []
jfutil.debug(True, 'Processing response from {}'.format(params['server']))
try:
if 'results' in data:
for result in data['results']:
id_list.append({
'process_id': result['id'],
'segment_id': result['segment_id']
})
except KeyError:
raise jfexceptions.UnexpectedResponseError(data)
# FOR EACH PROCESS ID AND SEGMENT ID PAIR...
events = []
if len(id_list) > 0:
jfutil.debug(
True,
'Fetching event details for {} processes'.format(len(id_list)))
event_cnt = 0
for i in id_list:
# USE THE IDS TO FORMAT A REST API URL AND FETCH THAT PROCESSES EVENT DETAILS
event_cnt = event_cnt + 1
event = {}
jfutil.debug(params['verbose'],
"Getting events for result {}".format(event_cnt))
url = "/api/v1/process/{}/0/event".format(i['process_id'])
response = send_query(params, url)
data = json.loads(response.read())
# IF THE USER REQUESTED MODLOADS...
if params['modloads'] == True:
# AND THERE ARE MODLOADS IN THE SEARCH RESULTS...
if 'modload_complete' in data['process']:
# EXTRACT THE PATHS OF THE EVENTS AND STORE THEM IN THE MODLOADS LIST
event['modloads'] = jfutil.get_event_paths(
data['process']['modload_complete'], 2, 'dir',
params['homogenize'])
else:
jfutil.debug(params['verbose'],
"Result {} has no modloads".format(event_cnt))
# IF THE USER REQUESTED FILEMODS...
if params['filemods'] == True:
# AND THERE ARE FILEMODS IN THE SEARCH RESULTS...
if 'filemod_complete' in data['process']:
# EXTRACT THE PATHS OF THE EVENTS AND STORE THEM IN THE FILEMODS LIST
event['filemods'] = jfutil.get_event_paths(
data['process']['filemod_complete'], 2, 'dir',
params['homogenize'])
else:
jfutil.debug(params['verbose'],
"Result {} has no filemods".format(event_cnt))
# IF THE USER REQUESTED REGMODS...
if params['regmods'] == True:
# AND THERE ARE REGMODS IN THE SEARCH RESULTS...
if 'regmod_complete' in data['process']:
# EXTRACT THE PATHS OF THE EVENTS AND STORE THEM IN THE REGMODS LIST
event['regmods'] = jfutil.get_event_paths(
data['process']['regmod_complete'], 2, 'reg',
params['homogenize'])
else:
jfutil.debug(params['verbose'],
"Result {} has no regmods".format(event_cnt))
# IF THE USER REQUESTED CHILDPROCS...
if params['childprocs'] == True:
# AND THERE ARE CHILDPROCS IN THE SEARCH RESULTS...
if 'childproc_complete' in data['process']:
# EXTRACT THE PATHS OF THE EVENTS AND STORE THEM IN THE CHILDPROCS LIST
event['childprocs'] = jfutil.get_event_paths(
data['process']['childproc_complete'], 3, 'dir',
params['homogenize'])
else:
jfutil.debug(
params['verbose'],
"Result {} has no childprocs".format(event_cnt))
# IF THE USER REQUESTED CROSSPROCS...
if params['crossprocs'] == True:
# AND THERE ARE CROSSPROCS IN THE SEARCH RESULTS...
if 'crossproc_complete' in data['process']:
# EXTRACT THE PATHS OF THE EVENTS AND STORE THEM IN THE CROSSPROCS LIST
event['crossprocs'] = jfutil.get_event_paths(
data['process']['crossproc_complete'], 4, 'dir',
params['homogenize'])
else:
jfutil.debug(
params['verbose'],
"Result {} has no crossprocs".format(event_cnt))
# IF THE USER REQUESTED NETCONNS...
if params['netconns'] == True:
# AND THERE ARE NETCONNS IN THE SEARCH RESULTS...
if 'netconn_complete' in data['process']:
# EXTRACT THE DOMAINS OF THE EVENTS AND STORE THEM IN THE NETCONNS LIST
event['netconns'] = jfutil.get_event_paths(
data['process']['netconn_complete'], 4, 'dir',
params['homogenize'])
else:
jfutil.debug(params['verbose'],
"Result {} has no netconns".format(event_cnt))
# IF ANY REQUESTED EVENTS HAVE BEEN EXTRACTED FROM THIS PROCESS, ADD THEM TO THE EVENT LIST
if 'modloads' in event or 'filemods' in event or 'regmods' in event or 'childprocs' in event or 'crossprocs' in event or 'netconns' in event:
events.append(event)
else:
raise jfexceptions.NoResultsError(query)
# IF ANY EVENTS WERE FOUND, RETURN THE EVENTS LIST
if len(events) > 0:
return events
else:
raise jfexceptions.NoEventsFoundError(query)