def manual_analysis(choice):
"""
Allows the execution of analysis functions with the manual flag. Output will mainly be in the Output window and some instances require user interaction.
:param choice: the manual function to be executed
"""
w = NotifyProgress('Address count')
w.show()
trace = prepare_trace()
func = manual_func[choice]
w.pbar_update(10)
func(deepcopy(trace), manual=True, update=w)
w.close()
def manual_analysis(choice):
"""
Allows the execution of analysis functions with the manual flag. Output will mainly be in the Output window and some instances require user interaction.
:param choice: the manual function to be executed
"""
w = NotifyProgress('Address count')
w.show()
trace = prepare_trace()
func = manual_func[choice]
w.pbar_update(10)
func(deepcopy(trace), manual=True, update=w)
w.close()
def clustering_analysis(visualization=0, grade=False, trace=None):
"""
Clustering analysis wrapper which clusters the trace into repeating instructions and presents the results in the Clustering Viewer.
:param visualization: output via Clustering Viewer or output window
:param grade: grading
:param trace: instruction trace
"""
if trace is None:
trace = prepare_trace()
w = NotifyProgress('Clustering')
w.show()
try:
try:
if not trace.constant_propagation:
trace = optimization_const_propagation(trace)
if not trace.stack_addr_propagation:
trace = optimization_stack_addr_propagation(trace)
except:
pass
w.pbar_update(30)
# cluster
vr = find_virtual_regs(deepcopy(trace))
w.pbar_update(20)
cluster = repetition_clustering(deepcopy(trace))
w.pbar_update(25)
if visualization == 0:
v0 = ClusterViewer(cluster,
create_bb_diff,
trace.ctx_reg_size,
save_func=save)
w.pbar_update(24)
v0.Show()
prev_ctx = defaultdict(lambda: 0)
stack_changes = defaultdict(lambda: 0)
for line in cluster:
if isinstance(line, Traceline):
prev_ctx = line.ctx
else:
stack_changes = create_cluster_gist(
line, trace.ctx_reg_size, prev_ctx, stack_changes)
prev_ctx = line[-1].ctx
# sort the stack_changes by address
sorted_result = sorted(stack_changes.keys())
sorted_result.reverse()
w.close()
v1 = StackChangeViewer(vr, sorted_result, stack_changes)
v1.Show()
else:
w.close()
visualize_cli(cluster)
except:
w.close()
def address_heuristic():
"""
Compute the occurrence of every address in the instruction trace.
"""
w = NotifyProgress('Address count')
w.show()
try:
trace = prepare_trace()
w.pbar_update(40)
ac = address_count(deepcopy(trace))
w.pbar_update(60)
w.close()
for addr, count in ac:
print 'Address %x (Disasm: %s) was encountered %s times.' % (addr, GetDisasm(addr), count)
except:
print '[*] An exception occurred! Quitting! '
w.close()
def address_heuristic():
"""
Compute the occurrence of every address in the instruction trace.
"""
w = NotifyProgress('Address count')
w.show()
try:
trace = prepare_trace()
w.pbar_update(40)
ac = address_count(deepcopy(trace))
w.pbar_update(60)
w.close()
for addr, count in ac:
print 'Address %x (Disasm: %s) was encountered %s times.' % (addr, GetDisasm(addr), count)
except:
print '[*] An exception occurred! Quitting! '
w.close()
def clustering_analysis(visualization=0, grade=False, trace=None):
"""
Clustering analysis wrapper which clusters the trace into repeating instructions and presents the results in the Clustering Viewer.
:param visualization: output via Clustering Viewer or output window
:param grade: grading
:param trace: instruction trace
"""
if trace is None:
trace = prepare_trace()
w = NotifyProgress('Clustering')
w.show()
try:
try:
if not trace.constant_propagation:
trace = optimization_const_propagation(trace)
if not trace.stack_addr_propagation:
trace = optimization_stack_addr_propagation(trace)
except:
pass
w.pbar_update(30)
# cluster
vr = find_virtual_regs(deepcopy(trace))
w.pbar_update(20)
cluster = repetition_clustering(deepcopy(trace))
w.pbar_update(25)
if visualization == 0:
v0 = ClusterViewer(cluster, create_bb_diff, trace.ctx_reg_size, save_func=save)
w.pbar_update(24)
v0.Show()
prev_ctx = defaultdict(lambda: 0)
stack_changes = defaultdict(lambda: 0)
for line in cluster:
if isinstance(line, Traceline):
prev_ctx = line.ctx
else:
stack_changes = create_cluster_gist(line, trace.ctx_reg_size, prev_ctx, stack_changes)
prev_ctx = line[-1].ctx
# sort the stack_changes by address
sorted_result = sorted(stack_changes.keys())
sorted_result.reverse()
w.close()
v1 = StackChangeViewer(vr, sorted_result, stack_changes)
v1.Show()
else:
w.close()
visualize_cli(cluster)
except:
w.close()
def grading_automaton(visualization=0):
"""
Grading System Analysis computes a grade for every trace line. It is basically a combination of all available analysis capabilities and runs them one after another, increases the grade
for those trace lines which are in the analysis result and then runs the next trace analysis. In between the analysis runs a pattern matching run is started, to increase / decrease cer-
tain trace lines grades based on known patterns. The patterns are modelled after known short comings of the analysis capabilities.
:param trace: instruction trace
:return: graded instruction trace
"""
vmr = get_vmr()
w = NotifyProgress('Grading')
w.show()
trace = prepare_trace()
orig_trace = deepcopy(trace)
try:
### INIT THE TRACE GRADES ###
trace = init_grading(deepcopy(trace))
w.pbar_update(10) # 10%
### REGISTER USAGE BASED: this must be done before optimization
reg_dict = defaultdict(lambda: 0)
# find the register infrastructure and vm addressing scheme -> this tells us which registers are used for addressing and are not important for grading_automaton
try:
for line in trace:
assert isinstance(line, Traceline)
if line.is_op2_reg and get_reg_class(line.disasm[2]) is not None: # get reg class will only return != None for the 8-16 standard cpu regs
reg_dict[get_reg_class(line.disasm[2])] += 1
# get the sorted list of regs highest occurence first
sorted_keys = sorted(reg_dict.items(), key=operator.itemgetter(1), reverse=True) # sorted_list = list of (reg_name, frequency)
length = len(sorted_keys)
w.pbar_update(10) # 20%
# classify the important and less important registers
if length % 2 == 0:
important_regs = set(reg[0] for reg in sorted_keys[:(length / 2)])
disregard_regs = set(reg[0] for reg in sorted_keys[(length / 2):])
else:
# if this is the case, one more register gets declared unimportant, since it is better to be more careful about raising grades
important_regs = set(reg[0] for reg in sorted_keys[:(length - 1) / 2])
disregard_regs = set(reg[0] for reg in sorted_keys[(length - 1) / 2:])
except:
pass
### OPTIMIZE TRACE ###
try:
if not trace.constant_propagation:
trace = optimization_const_propagation(trace)
except:
pass
w.pbar_update(10) #30%
try:
if not trace.stack_addr_propagation:
trace = optimization_stack_addr_propagation(trace)
except:
pass
### REGISTER USAGE AND INPUT OUTPUT BASED ###
# raise the grade of line containing input and output values
try:
values = find_input(deepcopy(trace)).union(find_output(deepcopy(trace)))
for line in trace:
for val in values:
if val in line.to_str_line():
line.raise_grade(vmr.in_out)
w.pbar_update(10) #40%
# backtrace regs and raise grade
virt_regs = find_virtual_regs(deepcopy(trace))
for key in virt_regs:
if get_reg_class(key) in important_regs:
for line in follow_virt_reg(deepcopy(trace), virt_reg_addr=virt_regs[key]):
try:
for other in trace:
if line == other:
other.raise_grade(vmr.in_out)
except ValueError:
print 'The line %s was not found in the trace, hence the grade could not be raised properly!' % line.to_str_line()
elif get_reg_class(key) in disregard_regs:
for line in follow_virt_reg(deepcopy(trace), virt_reg_addr=virt_regs[key]):
try:
for other in trace:
if line == other:
other.lower_grade(vmr.in_out)
except ValueError:
print 'The line %s was not found in the trace, hence the grade could not be lowered properly!' % line.to_str_line()
except:
pass
w.pbar_update(5) #45%
### REGISTER USAGE FREQUENCY BASED ###
try:
# lower the grades for the most commonly used registers
for line in trace:
assert isinstance(line, Traceline)
if line.is_op1_reg and get_reg_class(line.disasm[1]) is not None: # get reg class will only return != None for the 8-16 standard cpu regs
reg_dict[get_reg_class(line.disasm[1])] += 1
# get the sorted list of regs highest occurrence first
sorted_keys = sorted(reg_dict.items(), key=operator.itemgetter(1), reverse=True) # sorted_list = list of (reg_name, frequency)
length = len(sorted_keys)
w.pbar_update(5) #50%
# classify the less important registers
if length % 2 == 0:
disregard_regs = set(reg[0] for reg in sorted_keys[:(length / 2)])
else:
disregard_regs = set(reg[0] for reg in sorted_keys[:(length - 1) / 2])
for line in trace:
assert isinstance(line, Traceline)
if line.is_jmp or line.is_mov or line.is_pop or line.is_push or line.disasm[0].startswith('ret') or line.disasm[
0].startswith('inc') or line.disasm[0].startswith('lea'):
line.lower_grade(vmr.pa_ma)
elif len(line.disasm) > 1 and get_reg_class(line.disasm[1]) in disregard_regs:
line.lower_grade(vmr.pa_ma)
except:
pass
w.pbar_update(10) #60%
### CLUSTERING BASED ###
try:
# raise the grades of the unique lines after clustering
cluster_result = repetition_clustering(deepcopy(trace))
for line in cluster_result:
if isinstance(line, Traceline):
trace[trace.index(line)].raise_grade(vmr.clu)
except:
pass
w.pbar_update(10) #70%
### PEEPHOLE GRADING ###
try:
# peephole grading
for line in trace:
assert isinstance(line, Traceline)
if line.disasm[0] in ['pop', 'push', 'inc', 'dec', 'lea', 'test'] or line.disasm[0].startswith('c') or line.is_jmp or line.is_mov or line.disasm[0].startswith('r'):
line.lower_grade(vmr.pa_ma)
elif len(line.disasm) > 1 and get_reg_class(line.disasm[1]) > 4:
continue
else:
line.raise_grade(vmr.pa_ma)
except:
pass
w.pbar_update(10) #80%
### OPTIMIZATION BASED ###
try:
opti_trace = optimize(deepcopy(trace))
w.pbar_update(10) #90%
for line in opti_trace:
assert isinstance(line, Traceline)
try: # trace is heavily changed after optimization, might not find the trace line in the pre_op_trace
trace[trace.index(line)].raise_grade(vmr.pa_ma)
except:
pass
# additionally raise grade for every line that uses the memory and is not a mov
if line.disasm_len == 3 and line.is_op1_mem and not line.is_mov:
try:
trace[trace.index(line)].raise_grade(vmr.mem_use)
except:
pass
else:
trace[trace.index(line)].lower_grade(vmr.pa_ma)
except:
pass
w.pbar_update(5)
### STATIC OPTIMIZATION BASED ###
# TODO atm this is a little workaround to include the static analysis results
try:
comments = set(v_inst.split(' ')[0] for v_inst in [Comment(ea) for ea in range(vmr.code_start, vmr.code_end)] if v_inst is not None)
print comments
ins = [c.lstrip('v').split('_')[0] for c in comments]
for line in trace:
if line.disasm[0] in ins:
line.raise_grade(vmr.static)
except:
pass
w.pbar_update(5)
### RECURSION ###
try:
recursion = 0
vm_func = find_vm_addr(orig_trace)
for line in orig_trace:
if line.disasm[0].startswith('call') and line.disasm[1].__contains__(vm_func):
recursion = recursion + 1
except:
pass
w.close()
grades = set([line.grade for line in trace])
max_grade = max(grades)
# raise the trace lines grade containing calls to maximum grade
try:
# such nach call und vm_addr
for line in trace:
if line.disasm[0].startswith('call') and line.disasm[1].__contains__(vm_func):
line.grade = max_grade
elif line.disasm[1].__contains__('ss:') or line.disasm[2].__contains('ss:'):
line.grade = max_grade
except:
pass
if visualization == 0:
v = GradingViewer(trace, save=save)
v.Show()
else:
threshold = AskLong(1, 'There are a total of %s grades: %s. Specify a threshold which lines to display:' % (len(grades), ''.join('%s ' % c for c in grades)))
if threshold > -1:
for line in trace:
if line.grade >= threshold:
print line.grade, line.to_str_line()
except Exception, e:
w.close()
msg(e.message + '\n')
def input_output_analysis(manual=False):
"""
Input / Output analysis wrapper which computes the components of the output values of the VM function and allows for comparing these with the input arguments to the VM function.
Afterwards the results are presented in the VMInputOutputViewer.
:param manual: let user choose Function for input output analysis
"""
func_addr = None
if manual:
func_addr = ChooseFunction('Please select the function for black box analysis')
w = NotifyProgress('In/Out')
w.show()
trace = prepare_trace()
vmr = get_vmr()
# find relevant regs and operands
ctx = {}
try:
if func_addr is not None: # TODO enable input / output analysis of all functions
input = find_input(deepcopy(trace))
output = find_output(deepcopy(trace))
w.close()
else:
vr = DynamicAnalyzer(find_virtual_regs, trace)
w.pbar_update(10)
vr.start()
input = DynamicAnalyzer(find_input, trace)
w.pbar_update(10)
input.start()
output = DynamicAnalyzer(find_output, trace)
w.pbar_update(10)
output.start()
vr.join()
w.pbar_update(20)
vr = vr.get_result()
# create the trace excerpt for every relevant reg
for key in vr.keys():
if get_reg_class(key) is not None:
ctx[key] = follow_virt_reg(deepcopy(trace), virt_reg_addr=vr[key], real_reg_name=key)
vmr.vm_stack_reg_mapping = ctx
w.pbar_update(20)
input.join()
w.pbar_update(10)
output.join()
w.pbar_update(10)
w.close()
v = VMInputOuputViewer(input.get_result(), output.get_result(), ctx)
v.Show()
except:
w.close()
def grading_automaton(visualization=0):
"""
Grading System Analysis computes a grade for every trace line. It is basically a combination of all available analysis capabilities and runs them one after another, increases the grade
for those trace lines which are in the analysis result and then runs the next trace analysis. In between the analysis runs a pattern matching run is started, to increase / decrease cer-
tain trace lines grades based on known patterns. The patterns are modelled after known short comings of the analysis capabilities.
:param trace: instruction trace
:return: graded instruction trace
"""
vmr = get_vmr()
w = NotifyProgress('Grading')
w.show()
trace = prepare_trace()
orig_trace = deepcopy(trace)
try:
### INIT THE TRACE GRADES ###
trace = init_grading(deepcopy(trace))
w.pbar_update(10) # 10%
### REGISTER USAGE BASED: this must be done before optimization
reg_dict = defaultdict(lambda: 0)
# find the register infrastructure and vm addressing scheme -> this tells us which registers are used for addressing and are not important for grading_automaton
try:
for line in trace:
assert isinstance(line, Traceline)
if line.is_op2_reg and get_reg_class(line.disasm[2]) is not None: # get reg class will only return != None for the 8-16 standard cpu regs
reg_dict[get_reg_class(line.disasm[2])] += 1
# get the sorted list of regs highest occurence first
sorted_keys = sorted(reg_dict.items(), key=operator.itemgetter(1), reverse=True) # sorted_list = list of (reg_name, frequency)
length = len(sorted_keys)
w.pbar_update(10) # 20%
# classify the important and less important registers
if length % 2 == 0:
important_regs = set(reg[0] for reg in sorted_keys[:(length / 2)])
disregard_regs = set(reg[0] for reg in sorted_keys[(length / 2):])
else:
# if this is the case, one more register gets declared unimportant, since it is better to be more careful about raising grades
important_regs = set(reg[0] for reg in sorted_keys[:(length - 1) / 2])
disregard_regs = set(reg[0] for reg in sorted_keys[(length - 1) / 2:])
except:
pass
### OPTIMIZE TRACE ###
try:
if not trace.constant_propagation:
trace = optimization_const_propagation(trace)
except:
pass
w.pbar_update(10) #30%
try:
if not trace.stack_addr_propagation:
trace = optimization_stack_addr_propagation(trace)
except:
pass
### REGISTER USAGE AND INPUT OUTPUT BASED ###
# raise the grade of line containing input and output values
try:
values = find_input(deepcopy(trace)).union(find_output(deepcopy(trace)))
for line in trace:
for val in values:
if val in line.to_str_line():
line.raise_grade(vmr.in_out)
w.pbar_update(10) #40%
# backtrace regs and raise grade
virt_regs = find_virtual_regs(deepcopy(trace))
for key in virt_regs:
if get_reg_class(key) in important_regs:
for line in follow_virt_reg(deepcopy(trace), virt_reg_addr=virt_regs[key]):
try:
for other in trace:
if line == other:
other.raise_grade(vmr.in_out)
except ValueError:
print 'The line %s was not found in the trace, hence the grade could not be raised properly!' % line.to_str_line()
elif get_reg_class(key) in disregard_regs:
for line in follow_virt_reg(deepcopy(trace), virt_reg_addr=virt_regs[key]):
try:
for other in trace:
if line == other:
other.lower_grade(vmr.in_out)
except ValueError:
print 'The line %s was not found in the trace, hence the grade could not be lowered properly!' % line.to_str_line()
except:
pass
w.pbar_update(5) #45%
### REGISTER USAGE FREQUENCY BASED ###
try:
# lower the grades for the most commonly used registers
for line in trace:
assert isinstance(line, Traceline)
if line.is_op1_reg and get_reg_class(line.disasm[1]) is not None: # get reg class will only return != None for the 8-16 standard cpu regs
reg_dict[get_reg_class(line.disasm[1])] += 1
# get the sorted list of regs highest occurrence first
sorted_keys = sorted(reg_dict.items(), key=operator.itemgetter(1), reverse=True) # sorted_list = list of (reg_name, frequency)
length = len(sorted_keys)
w.pbar_update(5) #50%
# classify the less important registers
if length % 2 == 0:
disregard_regs = set(reg[0] for reg in sorted_keys[:(length / 2)])
else:
disregard_regs = set(reg[0] for reg in sorted_keys[:(length - 1) / 2])
for line in trace:
assert isinstance(line, Traceline)
if line.is_jmp or line.is_mov or line.is_pop or line.is_push or line.disasm[0].startswith('ret') or line.disasm[
0].startswith('inc') or line.disasm[0].startswith('lea'):
line.lower_grade(vmr.pa_ma)
elif len(line.disasm) > 1 and get_reg_class(line.disasm[1]) in disregard_regs:
line.lower_grade(vmr.pa_ma)
except:
pass
w.pbar_update(10) #60%
### CLUSTERING BASED ###
try:
# raise the grades of the unique lines after clustering
cluster_result = repetition_clustering(deepcopy(trace))
for line in cluster_result:
if isinstance(line, Traceline):
trace[trace.index(line)].raise_grade(vmr.clu)
except:
pass
w.pbar_update(10) #70%
### PEEPHOLE GRADING ###
try:
# peephole grading
for line in trace:
assert isinstance(line, Traceline)
if line.disasm[0] in ['pop', 'push', 'inc', 'dec', 'lea', 'test'] or line.disasm[0].startswith('c') or line.is_jmp or line.is_mov or line.disasm[0].startswith('r'):
line.lower_grade(vmr.pa_ma)
elif len(line.disasm) > 1 and get_reg_class(line.disasm[1]) > 4:
continue
else:
line.raise_grade(vmr.pa_ma)
except:
pass
w.pbar_update(10) #80%
### OPTIMIZATION BASED ###
try:
opti_trace = optimize(deepcopy(trace))
w.pbar_update(10) #90%
for line in opti_trace:
assert isinstance(line, Traceline)
try: # trace is heavily changed after optimization, might not find the trace line in the pre_op_trace
trace[trace.index(line)].raise_grade(vmr.pa_ma)
except:
pass
# additionally raise grade for every line that uses the memory and is not a mov
if line.disasm_len == 3 and line.is_op1_mem and not line.is_mov:
try:
trace[trace.index(line)].raise_grade(vmr.mem_use)
except:
pass
else:
trace[trace.index(line)].lower_grade(vmr.pa_ma)
except:
pass
w.pbar_update(5)
### STATIC OPTIMIZATION BASED ###
# TODO atm this is a little workaround to include the static analysis results
try:
comments = set(v_inst.split(' ')[0] for v_inst in [Comment(ea) for ea in range(vmr.code_start, vmr.code_end)] if v_inst is not None)
print comments
ins = [c.lstrip('v').split('_')[0] for c in comments]
for line in trace:
if line.disasm[0] in ins:
line.raise_grade(vmr.static)
except:
pass
w.pbar_update(5)
### RECURSION ###
try:
recursion = 0
vm_func = find_vm_addr(orig_trace)
for line in orig_trace:
if line.disasm[0].startswith('call') and line.disasm[1].__contains__(vm_func):
recursion = recursion + 1
except:
pass
w.close()
grades = set([line.grade for line in trace])
max_grade = max(grades)
# raise the trace lines grade containing calls to maximum grade
try:
# such nach call und vm_addr
for line in trace:
if line.disasm[0].startswith('call') and line.disasm[1].__contains__(vm_func):
line.grade = max_grade
elif line.disasm[1].__contains__('ss:') or line.disasm[2].__contains('ss:'):
line.grade = max_grade
except:
pass
if visualization == 0:
v = GradingViewer(trace, save=save)
v.Show()
else:
threshold = AskLong(1, 'There are a total of %s grades: %s. Specify a threshold which lines to display:' % (len(grades), ''.join('%s ' % c for c in grades)))
if threshold > -1:
for line in trace:
if line.grade >= threshold:
print line.grade, line.to_str_line()
except Exception, e:
w.close()
msg(e.message + '\n')
def input_output_analysis(manual=False):
"""
Input / Output analysis wrapper which computes the components of the output values of the VM function and allows for comparing these with the input arguments to the VM function.
Afterwards the results are presented in the VMInputOutputViewer.
:param manual: let user choose Function for input output analysis
"""
func_addr = None
if manual:
func_addr = ChooseFunction('Please select the function for black box analysis')
w = NotifyProgress('In/Out')
w.show()
trace = prepare_trace()
vmr = get_vmr()
# find relevant regs and operands
ctx = {}
try:
if func_addr is not None: # TODO enable input / output analysis of all functions
input = find_input(deepcopy(trace))
output = find_output(deepcopy(trace))
w.close()
else:
vr = DynamicAnalyzer(find_virtual_regs, trace)
w.pbar_update(10)
vr.start()
input = DynamicAnalyzer(find_input, trace)
w.pbar_update(10)
input.start()
output = DynamicAnalyzer(find_output, trace)
w.pbar_update(10)
output.start()
vr.join()
w.pbar_update(20)
vr = vr.get_result()
# create the trace excerpt for every relevant reg
for key in vr.keys():
if get_reg_class(key) is not None:
ctx[key] = follow_virt_reg(deepcopy(trace), virt_reg_addr=vr[key], real_reg_name=key)
vmr.vm_stack_reg_mapping = ctx
w.pbar_update(20)
input.join()
w.pbar_update(10)
output.join()
w.pbar_update(10)
w.close()
v = VMInputOuputViewer(input.get_result(), output.get_result(), ctx)
v.Show()
except:
w.close()