def Show():
settings = SettingsView()
settings.Compile()
vmr = get_vmr()
vm_ctx = vmr.vm_ctx
settings.iCodeStart.value = vm_ctx.code_start
settings.iCodeEnd.value = vm_ctx.code_end
settings.iBaseAddr.value = vm_ctx.base_addr
settings.iVMAddr.value = vm_ctx.vm_addr
settings.rGreedyCluster.checked = vmr.greedy
settings.rShowBB.checked = vmr.bb
settings.iClusterHeu.value = vmr.cluster_magic
settings.iInOut.value = vmr.in_out
settings.iClu.value = vmr.clu
settings.iPaMa.value = vmr.pa_ma
settings.iMeUs.value = vmr.mem_use
settings.iSta.value = vmr.static
settings.rStepInSysLibs.checked = vmr.sys_libs
settings.rFuncParams.checked = vmr.extract_param
if settings.Execute() == 0: # Cancel
settings.Free()
else: # Confirm
vmr = get_vmr()
# VM values
vm_ctx = VMContext()
vm_ctx.code_start = settings.iCodeStart.value
vm_ctx.code_end = settings.iCodeEnd.value
vm_ctx.base_addr = settings.iBaseAddr.value
vm_ctx.vm_addr = settings.iVMAddr.value
vmr.vm_ctx = vm_ctx
vmr.in_out = settings.iInOut.value
vmr.clu = settings.iClu.value
vmr.pa_ma = settings.iPaMa.value
vmr.mem_use = settings.iMeUs.value
vmr.static = settings.iSta.value
# Env values
vmr.sys_libs = settings.rStepInSysLibs.checked
vmr.extract_param = settings.rFuncParams.checked
vmr.greedy = settings.rGreedyCluster.checked
vmr.bb = settings.rShowBB.checked
vmr.cluster_magic = settings.iClusterHeu.value
settings.Free()
def Show():
settings = SettingsView()
settings.Compile()
vmr = get_vmr()
vm_ctx = vmr.vm_ctx
settings.iCodeStart.value = vm_ctx.code_start
settings.iCodeEnd.value = vm_ctx.code_end
settings.iBaseAddr.value = vm_ctx.base_addr
settings.iVMAddr.value = vm_ctx.vm_addr
settings.rGreedyCluster.checked = vmr.greedy
settings.rShowBB.checked = vmr.bb
settings.iClusterHeu.value = vmr.cluster_magic
settings.iInOut.value = vmr.in_out
settings.iClu.value = vmr.clu
settings.iPaMa.value = vmr.pa_ma
settings.iMeUs.value = vmr.mem_use
settings.iSta.value = vmr.static
settings.rStepInSysLibs.checked = vmr.sys_libs
settings.rFuncParams.checked = vmr.extract_param
if settings.Execute() == 0: # Cancel
settings.Free()
else: # Confirm
vmr = get_vmr()
# VM values
vm_ctx = VMContext()
vm_ctx.code_start = settings.iCodeStart.value
vm_ctx.code_end = settings.iCodeEnd.value
vm_ctx.base_addr = settings.iBaseAddr.value
vm_ctx.vm_addr = settings.iVMAddr.value
vmr.vm_ctx = vm_ctx
vmr.in_out = settings.iInOut.value
vmr.clu = settings.iClu.value
vmr.pa_ma = settings.iPaMa.value
vmr.mem_use = settings.iMeUs.value
vmr.static = settings.iSta.value
# Env values
vmr.sys_libs = settings.rStepInSysLibs.checked
vmr.extract_param = settings.rFuncParams.checked
vmr.greedy = settings.rGreedyCluster.checked
vmr.bb = settings.rShowBB.checked
vmr.cluster_magic = settings.iClusterHeu.value
settings.Free()
def find_virtual_regs(trace, manual=False, update=None):
"""
Maps the virtual registers on the stack to the actual registers after the vm exit.
:param trace: instruction trace
:return: virtual registers dict which maps the real regs onto virtual ones via stack addresses
"""
vmr = get_vmr()
assert isinstance(trace, Trace)
virt_regs = defaultdict(lambda: False)
# trace, vm_seg_start, vm_seg_end = extract_vm_segment(trace)
while trace:
try:
elem = trace.pop(len(trace) - 1)
if len(elem.disasm) > 0 and elem.disasm[0] == 'pop':
opnd = elem.disasm[1]
if get_reg_class(opnd) is None: # if not a register it is a mem_loc
pass
elif virt_regs[opnd]:
pass
else:
# the context always shows the registers after the execution, so we nee the SP from the instruction before
stack_addr = trace[len(trace) - 1].ctx[get_reg('rsp', trace.ctx_reg_size)]
virt_regs[opnd] = stack_addr
except:
pass
if update is not None:
update.pbar_update(60)
vmr.vm_stack_reg_mapping = virt_regs
if manual:
print ''.join('%s:%s\n' % (c, virt_regs[c]) for c in virt_regs.keys())
return virt_regs
def find_virtual_regs(trace, manual=False, update=None):
"""
Maps the virtual registers on the stack to the actual registers after the vm exit.
:param trace: instruction trace
:return: virtual registers dict which maps the real regs onto virtual ones via stack addresses
"""
vmr = get_vmr()
assert isinstance(trace, Trace)
virt_regs = defaultdict(lambda: False)
# trace, vm_seg_start, vm_seg_end = extract_vm_segment(trace)
while trace:
try:
elem = trace.pop(len(trace) - 1)
if len(elem.disasm) > 0 and elem.disasm[0] == 'pop':
opnd = elem.disasm[1]
if get_reg_class(opnd) is None: # if not a register it is a mem_loc
pass
elif virt_regs[opnd]:
pass
else:
# the context always shows the registers after the execution, so we nee the SP from the instruction before
stack_addr = trace[len(trace) - 1].ctx[get_reg('rsp', trace.ctx_reg_size)]
virt_regs[opnd] = stack_addr
except:
pass
if update is not None:
update.pbar_update(60)
vmr.vm_stack_reg_mapping = virt_regs
if manual:
print ''.join('%s:%s\n' % (c, virt_regs[c]) for c in virt_regs.keys())
return virt_regs
def OnCustomContextMenu(self, point):
menu = QtGui.QMenu()
init_index = self.treeView.indexAt(point)
index = self.treeView.indexAt(point)
level = 0
while index.parent().isValid():
index = index.parent()
level += 1
text = 'Remove Line'
if level == 0:
text = "Remove Cluster / Line"
elif level == 1 and get_vmr().bb:
text = "Remove Basic Block"
elif level == 2:
text = "Remove Line"
try:
action_remove = QtGui.QAction(
text,
self.treeView,
triggered=lambda: self.ItemDoubleClickSlot(init_index))
menu.addAction(action_remove)
menu.addSeparator()
except:
print '[*] An Exception occured, remove action could not be added to the menu!'
# Actions
action_remove_threshold = QtGui.QAction(
'Remove several clusters...',
self.treeView,
triggered=lambda: self.ClusterRemoval())
action_undo = QtGui.QAction('Undo',
self.treeView,
triggered=lambda: self.Undo())
action_restore = QtGui.QAction('Restore original trace',
self.treeView,
triggered=lambda: self.Restore())
action_export_trace = QtGui.QAction('Export this trace ...',
self.treeView,
triggered=lambda: self.SaveTrace())
action_close_viewer = QtGui.QAction('Close Viewer',
self.treeView,
triggered=lambda: self.Close(4))
# add actions to menu
menu.addAction(action_remove_threshold)
menu.addAction(action_undo)
menu.addAction(action_restore)
menu.addAction(action_export_trace)
menu.addSeparator()
menu.addAction(action_close_viewer)
menu.exec_(self.treeView.viewport().mapToGlobal(point))
def input_output_analysis(manual=False):
"""
Input / Output analysis wrapper which computes the components of the output values of the VM function and allows for comparing these with the input arguments to the VM function.
Afterwards the results are presented in the VMInputOutputViewer.
:param manual: let user choose Function for input output analysis
"""
func_addr = None
if manual:
func_addr = ChooseFunction(
'Please select the function for black box analysis')
w = NotifyProgress('In/Out')
w.show()
trace = prepare_trace()
vmr = get_vmr()
# find relevant regs and operands
ctx = {}
try:
if func_addr is not None: # TODO enable input / output analysis of all functions
input = find_input(deepcopy(trace))
output = find_output(deepcopy(trace))
w.close()
else:
vr = DynamicAnalyzer(find_virtual_regs, trace)
w.pbar_update(10)
vr.start()
input = DynamicAnalyzer(find_input, trace)
w.pbar_update(10)
input.start()
output = DynamicAnalyzer(find_output, trace)
w.pbar_update(10)
output.start()
vr.join()
w.pbar_update(20)
vr = vr.get_result()
# create the trace excerpt for every relevant reg
for key in vr.keys():
if get_reg_class(key) is not None:
ctx[key] = follow_virt_reg(deepcopy(trace),
virt_reg_addr=vr[key],
real_reg_name=key)
vmr.vm_stack_reg_mapping = ctx
w.pbar_update(20)
input.join()
w.pbar_update(10)
output.join()
w.pbar_update(10)
w.close()
v = VMInputOuputViewer(input.get_result(), output.get_result(),
ctx)
v.Show()
except:
w.close()
def dynamic_vmctx(manual=False):
"""
Compute the VM context values based on the trace.
:param manual: output in output window
"""
trace = prepare_trace()
vm_ctx = dynamic_vm_values(trace)
vmr = get_vmr()
vmr.vm_ctx = vm_ctx
if manual:
print 'Code Start: %x; Code End: %x; Base Addr: %x; VM Addr: %x' % (vm_ctx.code_start, vm_ctx.code_end, vm_ctx.base_addr, vm_ctx.vm_addr)
def dynamic_vmctx(manual=False):
"""
Compute the VM context values based on the trace.
:param manual: output in output window
"""
trace = prepare_trace()
vm_ctx = dynamic_vm_values(trace)
vmr = get_vmr()
vmr.vm_ctx = vm_ctx
if manual:
print 'Code Start: %x; Code End: %x; Base Addr: %x; VM Addr: %x' % (vm_ctx.code_start, vm_ctx.code_end, vm_ctx.base_addr, vm_ctx.vm_addr)
def gen_instruction_trace(choice):
"""
Generate instruction trace
:param choice: which debugger to use
"""
dbg_handl = get_dh(choice)
vmr = get_vmr()
trace = dbg_handl.gen_instruction_trace()
if trace is not None:
vmr.trace = trace
else:
raise Exception('[*] Trace seems to be None, so it was disregarded!')
def gen_instruction_trace(choice):
"""
Generate instruction trace
:param choice: which debugger to use
"""
dbg_handl = get_dh(choice)
vmr = get_vmr()
trace = dbg_handl.gen_instruction_trace()
if trace is not None:
vmr.trace = trace
else:
raise Exception('[*] Trace seems to be None, so it was disregarded!')
def static_deobfuscate(display=0, userchoice=False):
"""
Wrapper for deobfuscate function which allows for manual user Input.
:param display: Bool -> use output window or BBGraphViewer
:param userchoice: let user input vm context values
"""
vmr = get_vmr()
if vmr.code_start == BADADDR:
try:
vm_ctx = static_vmctx()
vmr.vm_ctx = vm_ctx
except Exception, e:
print e.message
print e.args
def input_output_analysis(manual=False):
"""
Input / Output analysis wrapper which computes the components of the output values of the VM function and allows for comparing these with the input arguments to the VM function.
Afterwards the results are presented in the VMInputOutputViewer.
:param manual: let user choose Function for input output analysis
"""
func_addr = None
if manual:
func_addr = ChooseFunction('Please select the function for black box analysis')
w = NotifyProgress('In/Out')
w.show()
trace = prepare_trace()
vmr = get_vmr()
# find relevant regs and operands
ctx = {}
try:
if func_addr is not None: # TODO enable input / output analysis of all functions
input = find_input(deepcopy(trace))
output = find_output(deepcopy(trace))
w.close()
else:
vr = DynamicAnalyzer(find_virtual_regs, trace)
w.pbar_update(10)
vr.start()
input = DynamicAnalyzer(find_input, trace)
w.pbar_update(10)
input.start()
output = DynamicAnalyzer(find_output, trace)
w.pbar_update(10)
output.start()
vr.join()
w.pbar_update(20)
vr = vr.get_result()
# create the trace excerpt for every relevant reg
for key in vr.keys():
if get_reg_class(key) is not None:
ctx[key] = follow_virt_reg(deepcopy(trace), virt_reg_addr=vr[key], real_reg_name=key)
vmr.vm_stack_reg_mapping = ctx
w.pbar_update(20)
input.join()
w.pbar_update(10)
output.join()
w.pbar_update(10)
w.close()
v = VMInputOuputViewer(input.get_result(), output.get_result(), ctx)
v.Show()
except:
w.close()
def gen_trace(self, trace_start=BeginEA(), trace_end=BADADDR):
"""
主动生成trace
Generate trace for the loaded binary.
:param trace_start:
:param trace_end:
:return:
"""
vmr = get_vmr()
self.trace_init()
# reset color
heads = Heads(SegStart(ScreenEA()), SegEnd(ScreenEA()))
for i in heads:
SetColor(i, CIC_ITEM, 0xFFFFFF)
# start exec
RunTo(BeginEA())
event = GetDebuggerEvent(WFNE_SUSP, -1)
# enable tracing
EnableTracing(TRACE_STEP, 1)
if vmr.sys_libs:
pass
event = GetDebuggerEvent(WFNE_ANY | WFNE_CONT, -1)
while True:
event = GetDebuggerEvent(WFNE_ANY, -1)
addr = GetEventEa()
# change color of executed line
current_color = GetColor(addr, CIC_ITEM)
new_color = self.get_new_color(current_color)
SetColor(addr, CIC_ITEM, new_color)
# break by exception
if event <= 1:
break
# standardize the difference between ida_trace.txt files and generated trace files by debugger hook:
# since dbg_trace returns the cpu context before the instruction execution and trace files the ctx after
for line in self.trace:
try:
line.ctx = self.trace[self.trace.index(line) + 1].ctx
except IndexError:
line.ctx = defaultdict(lambda: '0')
# return the trace, for population see dbg_trace() below
msg('[*] Trace generated!\n')
if vmr.extract_param:
vmr.func_args = self.func_args
for key in self.func_args.keys():
print 'Function %s call args:' % key, ''.join(
'%s, ' % arg for arg in self.func_args[key]).rstrip(', ')
return self.trace
def repetition_cluster_round(cluster_list):
"""
One round of repetition cluster analysis.
:param cluster_list: list of clusters
:return: cluster list
"""
vmr = get_vmr()
assert isinstance(cluster_list, list)
test_length = len_check(cluster_list)
temp_cluster = [[cluster_list[cluster], cluster_list[cluster + 1]]
for cluster in range(0,
len(cluster_list) - 1, 2)]
# each tupel is tested for validity
for cluster in temp_cluster:
if cluster_list.count(cluster[0]) == cluster_list.count(cluster[1]):
occurence = 0
pop_indexes = []
try:
for j in range(len(cluster_list) - 1):
# if they are adjacent, they are labeled valid
if get_addr(cluster_list[j]) == get_addr(
cluster[0]) and get_addr(
cluster_list[j + 1]) == get_addr(cluster[1]):
pop_indexes.append(j + 1)
occurence += 1
if occurence > vmr.cluster_magic: # if validity occured more than once we have a new cluster
pop_ctr = 0
for ind in pop_indexes:
addition = cluster_list.pop(ind - pop_ctr)
pop_ctr += 1
base = cluster_list[ind - pop_ctr]
if isinstance(base, Traceline):
if isinstance(addition, Traceline):
cluster_list[ind - pop_ctr] = [base, addition]
elif isinstance(addition, list):
cluster_list[ind - pop_ctr] = [base] + addition
elif isinstance(base, list):
if isinstance(addition, Traceline):
cluster_list[ind - pop_ctr].append(addition)
elif isinstance(addition, list):
cluster_list[ind - pop_ctr].extend(addition)
except Exception, e:
print e.message
pass
def find_input(trace, manual=False, update=None):
"""
Find input operands to the vm_function.
:param trace: instruciton trace
:param manual: console output z/n
:return: a set of operands to the vm_function
"""
vmr = get_vmr()
if vmr.func_args:
func = GetFunctionName(find_vm_addr(deepcopy(trace)))
func_args = vmr.func_args[func]
ops = set()
if update is not None:
update.pbar_update(20)
ex_trace, vmp_seg_start, vmp_seg_end = extract_vm_segment(
deepcopy(trace)
) # use deepcopy trace, since we need the full one for find_ops_callconv
if update is not None:
update.pbar_update(20)
for line in ex_trace:
try:
# case inst reg, ss:[reg]
op = line.disasm[2]
# following is ida only
if op.startswith('ss:'):
# get the reg value from ctx
op = line.ctx[get_reg(line.disasm[1], trace.ctx_reg_size)]
ops.add(op.upper())
except:
pass
try:
# if we find the .vmp Segment addr or vm-function addr we should check the stack
for op in find_ops_callconv(trace, vmp_seg_start, vmp_seg_end):
ops.add(op.upper()) # set will eliminate double entries
if update is not None:
update.pbar_update(30)
for op in func_args:
ops.add(op.upper())
except:
pass
if update is not None:
update.pbar_update(10)
if manual:
print 'operands: %s' % ''.join('%s | ' % op for op in ops)
return ops
def OnCustomContextMenu(self, point):
menu = QtWidgets.QMenu()
init_index = self.treeView.indexAt(point)
index = self.treeView.indexAt(point)
level = 0
while index.parent().isValid():
index = index.parent()
level += 1
text = 'Remove Line'
if level == 0:
text = "Remove Cluster / Line"
elif level == 1 and get_vmr().bb:
text = "Remove Basic Block"
elif level == 2:
text = "Remove Line"
try:
action_remove = QtWidgets.QAction(text, self.treeView)
action_remove.triggered.connect(lambda: self.ItemDoubleClickSlot(init_index))
menu.addAction(action_remove)
menu.addSeparator()
except:
print '[*] An Exception occured, remove action could not be added to the menu!'
# Actions
action_remove_threshold = QtWidgets.QAction('Remove several clusters...', self.treeView)
action_remove_threshold.triggered.connect(self.ClusterRemoval)
action_undo = QtWidgets.QAction('Undo', self.treeView)
action_undo.triggered.connect(self.Undo)
action_restore = QtWidgets.QAction('Restore original trace', self.treeView)
action_restore.triggered.connect(self.Restore)
action_export_trace = QtWidgets.QAction('Export this trace ...', self.treeView)
action_export_trace.triggered.connect(self.SaveTrace)
action_close_viewer = QtWidgets.QAction('Close Viewer', self.treeView)
action_close_viewer.triggered.connect(lambda: self.Close(4))
# add actions to menu
menu.addAction(action_remove_threshold)
menu.addAction(action_undo)
menu.addAction(action_restore)
menu.addAction(action_export_trace)
menu.addSeparator()
menu.addAction(action_close_viewer)
menu.exec_(self.treeView.viewport().mapToGlobal(point))
def repetition_cluster_round(cluster_list):
"""
One round of repetition cluster analysis.
:param cluster_list: list of clusters
:return: cluster list
"""
vmr = get_vmr()
assert isinstance(cluster_list, list)
test_length = len_check(cluster_list)
temp_cluster = [[cluster_list[cluster], cluster_list[cluster + 1]] for cluster in
range(0, len(cluster_list) - 1, 2)]
# each tupel is tested for validity
for cluster in temp_cluster:
if cluster_list.count(cluster[0]) == cluster_list.count(cluster[1]):
occurence = 0
pop_indexes = []
try:
for j in range(len(cluster_list) - 1):
# if they are adjacent, they are labeled valid
if get_addr(cluster_list[j]) == get_addr(cluster[0]) and get_addr(cluster_list[j + 1]) == get_addr(
cluster[1]):
pop_indexes.append(j + 1)
occurence += 1
if occurence > vmr.cluster_magic: # if validity occured more than once we have a new cluster
pop_ctr = 0
for ind in pop_indexes:
addition = cluster_list.pop(ind - pop_ctr)
pop_ctr += 1
base = cluster_list[ind - pop_ctr]
if isinstance(base, Traceline):
if isinstance(addition, Traceline):
cluster_list[ind - pop_ctr] = [base, addition]
elif isinstance(addition, list):
cluster_list[ind - pop_ctr] = [base] + addition
elif isinstance(base, list):
if isinstance(addition, Traceline):
cluster_list[ind - pop_ctr].append(addition)
elif isinstance(addition, list):
cluster_list[ind - pop_ctr].extend(addition)
except Exception, e:
print e.message
pass
def find_input(trace, manual=False, update=None):
"""
Find input operands to the vm_function.
:param trace: instruciton trace
:param manual: console output z/n
:return: a set of operands to the vm_function
"""
vmr = get_vmr()
if vmr.func_args:
func = GetFunctionName(find_vm_addr(deepcopy(trace)))
func_args = vmr.func_args[func]
ops = set()
if update is not None:
update.pbar_update(20)
ex_trace, vmp_seg_start, vmp_seg_end = extract_vm_segment(deepcopy(trace)) # use deepcopy trace, since we need the full one for find_ops_callconv
if update is not None:
update.pbar_update(20)
for line in ex_trace:
try:
# case inst reg, ss:[reg]
op = line.disasm[2]
# following is ida only
if op.startswith('ss:'):
# get the reg value from ctx
op = line.ctx[get_reg(line.disasm[1], trace.ctx_reg_size)]
ops.add(op.upper())
except:
pass
try:
# if we find the .vmp Segment addr or vm-function addr we should check the stack
for op in find_ops_callconv(trace, vmp_seg_start, vmp_seg_end):
ops.add(op.upper()) # set will eliminate double entries
if update is not None:
update.pbar_update(30)
for op in func_args:
ops.add(op.upper())
except:
pass
if update is not None:
update.pbar_update(10)
if manual:
print 'operands: %s' % ''.join('%s | ' % op for op in ops)
return ops
def prepare_trace():
vmr = get_vmr()
if vmr.trace is None:
vmr.trace = load()
return deepcopy(vmr.trace)
def grading_automaton(visualization=0):
"""
Grading System Analysis computes a grade for every trace line. It is basically a combination of all available analysis capabilities and runs them one after another, increases the grade
for those trace lines which are in the analysis result and then runs the next trace analysis. In between the analysis runs a pattern matching run is started, to increase / decrease cer-
tain trace lines grades based on known patterns. The patterns are modelled after known short comings of the analysis capabilities.
:param trace: instruction trace
:return: graded instruction trace
"""
vmr = get_vmr()
w = NotifyProgress('Grading')
w.show()
trace = prepare_trace()
orig_trace = deepcopy(trace)
try:
### INIT THE TRACE GRADES ###
trace = init_grading(deepcopy(trace))
w.pbar_update(10) # 10%
### REGISTER USAGE BASED: this must be done before optimization
reg_dict = defaultdict(lambda: 0)
# find the register infrastructure and vm addressing scheme -> this tells us which registers are used for addressing and are not important for grading_automaton
try:
for line in trace:
assert isinstance(line, Traceline)
if line.is_op2_reg and get_reg_class(line.disasm[2]) is not None: # get reg class will only return != None for the 8-16 standard cpu regs
reg_dict[get_reg_class(line.disasm[2])] += 1
# get the sorted list of regs highest occurence first
sorted_keys = sorted(reg_dict.items(), key=operator.itemgetter(1), reverse=True) # sorted_list = list of (reg_name, frequency)
length = len(sorted_keys)
w.pbar_update(10) # 20%
# classify the important and less important registers
if length % 2 == 0:
important_regs = set(reg[0] for reg in sorted_keys[:(length / 2)])
disregard_regs = set(reg[0] for reg in sorted_keys[(length / 2):])
else:
# if this is the case, one more register gets declared unimportant, since it is better to be more careful about raising grades
important_regs = set(reg[0] for reg in sorted_keys[:(length - 1) / 2])
disregard_regs = set(reg[0] for reg in sorted_keys[(length - 1) / 2:])
except:
pass
### OPTIMIZE TRACE ###
try:
if not trace.constant_propagation:
trace = optimization_const_propagation(trace)
except:
pass
w.pbar_update(10) #30%
try:
if not trace.stack_addr_propagation:
trace = optimization_stack_addr_propagation(trace)
except:
pass
### REGISTER USAGE AND INPUT OUTPUT BASED ###
# raise the grade of line containing input and output values
try:
values = find_input(deepcopy(trace)).union(find_output(deepcopy(trace)))
for line in trace:
for val in values:
if val in line.to_str_line():
line.raise_grade(vmr.in_out)
w.pbar_update(10) #40%
# backtrace regs and raise grade
virt_regs = find_virtual_regs(deepcopy(trace))
for key in virt_regs:
if get_reg_class(key) in important_regs:
for line in follow_virt_reg(deepcopy(trace), virt_reg_addr=virt_regs[key]):
try:
for other in trace:
if line == other:
other.raise_grade(vmr.in_out)
except ValueError:
print 'The line %s was not found in the trace, hence the grade could not be raised properly!' % line.to_str_line()
elif get_reg_class(key) in disregard_regs:
for line in follow_virt_reg(deepcopy(trace), virt_reg_addr=virt_regs[key]):
try:
for other in trace:
if line == other:
other.lower_grade(vmr.in_out)
except ValueError:
print 'The line %s was not found in the trace, hence the grade could not be lowered properly!' % line.to_str_line()
except:
pass
w.pbar_update(5) #45%
### REGISTER USAGE FREQUENCY BASED ###
try:
# lower the grades for the most commonly used registers
for line in trace:
assert isinstance(line, Traceline)
if line.is_op1_reg and get_reg_class(line.disasm[1]) is not None: # get reg class will only return != None for the 8-16 standard cpu regs
reg_dict[get_reg_class(line.disasm[1])] += 1
# get the sorted list of regs highest occurrence first
sorted_keys = sorted(reg_dict.items(), key=operator.itemgetter(1), reverse=True) # sorted_list = list of (reg_name, frequency)
length = len(sorted_keys)
w.pbar_update(5) #50%
# classify the less important registers
if length % 2 == 0:
disregard_regs = set(reg[0] for reg in sorted_keys[:(length / 2)])
else:
disregard_regs = set(reg[0] for reg in sorted_keys[:(length - 1) / 2])
for line in trace:
assert isinstance(line, Traceline)
if line.is_jmp or line.is_mov or line.is_pop or line.is_push or line.disasm[0].startswith('ret') or line.disasm[
0].startswith('inc') or line.disasm[0].startswith('lea'):
line.lower_grade(vmr.pa_ma)
elif len(line.disasm) > 1 and get_reg_class(line.disasm[1]) in disregard_regs:
line.lower_grade(vmr.pa_ma)
except:
pass
w.pbar_update(10) #60%
### CLUSTERING BASED ###
try:
# raise the grades of the unique lines after clustering
cluster_result = repetition_clustering(deepcopy(trace))
for line in cluster_result:
if isinstance(line, Traceline):
trace[trace.index(line)].raise_grade(vmr.clu)
except:
pass
w.pbar_update(10) #70%
### PEEPHOLE GRADING ###
try:
# peephole grading
for line in trace:
assert isinstance(line, Traceline)
if line.disasm[0] in ['pop', 'push', 'inc', 'dec', 'lea', 'test'] or line.disasm[0].startswith('c') or line.is_jmp or line.is_mov or line.disasm[0].startswith('r'):
line.lower_grade(vmr.pa_ma)
elif len(line.disasm) > 1 and get_reg_class(line.disasm[1]) > 4:
continue
else:
line.raise_grade(vmr.pa_ma)
except:
pass
w.pbar_update(10) #80%
### OPTIMIZATION BASED ###
try:
opti_trace = optimize(deepcopy(trace))
w.pbar_update(10) #90%
for line in opti_trace:
assert isinstance(line, Traceline)
try: # trace is heavily changed after optimization, might not find the trace line in the pre_op_trace
trace[trace.index(line)].raise_grade(vmr.pa_ma)
except:
pass
# additionally raise grade for every line that uses the memory and is not a mov
if line.disasm_len == 3 and line.is_op1_mem and not line.is_mov:
try:
trace[trace.index(line)].raise_grade(vmr.mem_use)
except:
pass
else:
trace[trace.index(line)].lower_grade(vmr.pa_ma)
except:
pass
w.pbar_update(5)
### STATIC OPTIMIZATION BASED ###
# TODO atm this is a little workaround to include the static analysis results
try:
comments = set(v_inst.split(' ')[0] for v_inst in [Comment(ea) for ea in range(vmr.code_start, vmr.code_end)] if v_inst is not None)
print comments
ins = [c.lstrip('v').split('_')[0] for c in comments]
for line in trace:
if line.disasm[0] in ins:
line.raise_grade(vmr.static)
except:
pass
w.pbar_update(5)
### RECURSION ###
try:
recursion = 0
vm_func = find_vm_addr(orig_trace)
for line in orig_trace:
if line.disasm[0].startswith('call') and line.disasm[1].__contains__(vm_func):
recursion = recursion + 1
except:
pass
w.close()
grades = set([line.grade for line in trace])
max_grade = max(grades)
# raise the trace lines grade containing calls to maximum grade
try:
# such nach call und vm_addr
for line in trace:
if line.disasm[0].startswith('call') and line.disasm[1].__contains__(vm_func):
line.grade = max_grade
elif line.disasm[1].__contains__('ss:') or line.disasm[2].__contains('ss:'):
line.grade = max_grade
except:
pass
if visualization == 0:
v = GradingViewer(trace, save=save)
v.Show()
else:
threshold = AskLong(1, 'There are a total of %s grades: %s. Specify a threshold which lines to display:' % (len(grades), ''.join('%s ' % c for c in grades)))
if threshold > -1:
for line in trace:
if line.grade >= threshold:
print line.grade, line.to_str_line()
except Exception, e:
w.close()
msg(e.message + '\n')
def grading_automaton(visualization=0):
"""
Grading System Analysis computes a grade for every trace line. It is basically a combination of all available analysis capabilities and runs them one after another, increases the grade
for those trace lines which are in the analysis result and then runs the next trace analysis. In between the analysis runs a pattern matching run is started, to increase / decrease cer-
tain trace lines grades based on known patterns. The patterns are modelled after known short comings of the analysis capabilities.
:param trace: instruction trace
:return: graded instruction trace
"""
vmr = get_vmr()
w = NotifyProgress('Grading')
w.show()
trace = prepare_trace()
orig_trace = deepcopy(trace)
try:
### INIT THE TRACE GRADES ###
trace = init_grading(deepcopy(trace))
w.pbar_update(10) # 10%
### REGISTER USAGE BASED: this must be done before optimization
reg_dict = defaultdict(lambda: 0)
# find the register infrastructure and vm addressing scheme -> this tells us which registers are used for addressing and are not important for grading_automaton
try:
for line in trace:
assert isinstance(line, Traceline)
if line.is_op2_reg and get_reg_class(line.disasm[2]) is not None: # get reg class will only return != None for the 8-16 standard cpu regs
reg_dict[get_reg_class(line.disasm[2])] += 1
# get the sorted list of regs highest occurence first
sorted_keys = sorted(reg_dict.items(), key=operator.itemgetter(1), reverse=True) # sorted_list = list of (reg_name, frequency)
length = len(sorted_keys)
w.pbar_update(10) # 20%
# classify the important and less important registers
if length % 2 == 0:
important_regs = set(reg[0] for reg in sorted_keys[:(length / 2)])
disregard_regs = set(reg[0] for reg in sorted_keys[(length / 2):])
else:
# if this is the case, one more register gets declared unimportant, since it is better to be more careful about raising grades
important_regs = set(reg[0] for reg in sorted_keys[:(length - 1) / 2])
disregard_regs = set(reg[0] for reg in sorted_keys[(length - 1) / 2:])
except:
pass
### OPTIMIZE TRACE ###
try:
if not trace.constant_propagation:
trace = optimization_const_propagation(trace)
except:
pass
w.pbar_update(10) #30%
try:
if not trace.stack_addr_propagation:
trace = optimization_stack_addr_propagation(trace)
except:
pass
### REGISTER USAGE AND INPUT OUTPUT BASED ###
# raise the grade of line containing input and output values
try:
values = find_input(deepcopy(trace)).union(find_output(deepcopy(trace)))
for line in trace:
for val in values:
if val in line.to_str_line():
line.raise_grade(vmr.in_out)
w.pbar_update(10) #40%
# backtrace regs and raise grade
virt_regs = find_virtual_regs(deepcopy(trace))
for key in virt_regs:
if get_reg_class(key) in important_regs:
for line in follow_virt_reg(deepcopy(trace), virt_reg_addr=virt_regs[key]):
try:
for other in trace:
if line == other:
other.raise_grade(vmr.in_out)
except ValueError:
print 'The line %s was not found in the trace, hence the grade could not be raised properly!' % line.to_str_line()
elif get_reg_class(key) in disregard_regs:
for line in follow_virt_reg(deepcopy(trace), virt_reg_addr=virt_regs[key]):
try:
for other in trace:
if line == other:
other.lower_grade(vmr.in_out)
except ValueError:
print 'The line %s was not found in the trace, hence the grade could not be lowered properly!' % line.to_str_line()
except:
pass
w.pbar_update(5) #45%
### REGISTER USAGE FREQUENCY BASED ###
try:
# lower the grades for the most commonly used registers
for line in trace:
assert isinstance(line, Traceline)
if line.is_op1_reg and get_reg_class(line.disasm[1]) is not None: # get reg class will only return != None for the 8-16 standard cpu regs
reg_dict[get_reg_class(line.disasm[1])] += 1
# get the sorted list of regs highest occurrence first
sorted_keys = sorted(reg_dict.items(), key=operator.itemgetter(1), reverse=True) # sorted_list = list of (reg_name, frequency)
length = len(sorted_keys)
w.pbar_update(5) #50%
# classify the less important registers
if length % 2 == 0:
disregard_regs = set(reg[0] for reg in sorted_keys[:(length / 2)])
else:
disregard_regs = set(reg[0] for reg in sorted_keys[:(length - 1) / 2])
for line in trace:
assert isinstance(line, Traceline)
if line.is_jmp or line.is_mov or line.is_pop or line.is_push or line.disasm[0].startswith('ret') or line.disasm[
0].startswith('inc') or line.disasm[0].startswith('lea'):
line.lower_grade(vmr.pa_ma)
elif len(line.disasm) > 1 and get_reg_class(line.disasm[1]) in disregard_regs:
line.lower_grade(vmr.pa_ma)
except:
pass
w.pbar_update(10) #60%
### CLUSTERING BASED ###
try:
# raise the grades of the unique lines after clustering
cluster_result = repetition_clustering(deepcopy(trace))
for line in cluster_result:
if isinstance(line, Traceline):
trace[trace.index(line)].raise_grade(vmr.clu)
except:
pass
w.pbar_update(10) #70%
### PEEPHOLE GRADING ###
try:
# peephole grading
for line in trace:
assert isinstance(line, Traceline)
if line.disasm[0] in ['pop', 'push', 'inc', 'dec', 'lea', 'test'] or line.disasm[0].startswith('c') or line.is_jmp or line.is_mov or line.disasm[0].startswith('r'):
line.lower_grade(vmr.pa_ma)
elif len(line.disasm) > 1 and get_reg_class(line.disasm[1]) > 4:
continue
else:
line.raise_grade(vmr.pa_ma)
except:
pass
w.pbar_update(10) #80%
### OPTIMIZATION BASED ###
try:
opti_trace = optimize(deepcopy(trace))
w.pbar_update(10) #90%
for line in opti_trace:
assert isinstance(line, Traceline)
try: # trace is heavily changed after optimization, might not find the trace line in the pre_op_trace
trace[trace.index(line)].raise_grade(vmr.pa_ma)
except:
pass
# additionally raise grade for every line that uses the memory and is not a mov
if line.disasm_len == 3 and line.is_op1_mem and not line.is_mov:
try:
trace[trace.index(line)].raise_grade(vmr.mem_use)
except:
pass
else:
trace[trace.index(line)].lower_grade(vmr.pa_ma)
except:
pass
w.pbar_update(5)
### STATIC OPTIMIZATION BASED ###
# TODO atm this is a little workaround to include the static analysis results
try:
comments = set(v_inst.split(' ')[0] for v_inst in [Comment(ea) for ea in range(vmr.code_start, vmr.code_end)] if v_inst is not None)
print comments
ins = [c.lstrip('v').split('_')[0] for c in comments]
for line in trace:
if line.disasm[0] in ins:
line.raise_grade(vmr.static)
except:
pass
w.pbar_update(5)
### RECURSION ###
try:
recursion = 0
vm_func = find_vm_addr(orig_trace)
for line in orig_trace:
if line.disasm[0].startswith('call') and line.disasm[1].__contains__(vm_func):
recursion = recursion + 1
except:
pass
w.close()
grades = set([line.grade for line in trace])
max_grade = max(grades)
# raise the trace lines grade containing calls to maximum grade
try:
# such nach call und vm_addr
for line in trace:
if line.disasm[0].startswith('call') and line.disasm[1].__contains__(vm_func):
line.grade = max_grade
elif line.disasm[1].__contains__('ss:') or line.disasm[2].__contains('ss:'):
line.grade = max_grade
except:
pass
if visualization == 0:
v = GradingViewer(trace, save=save)
v.Show()
else:
threshold = AskLong(1, 'There are a total of %s grades: %s. Specify a threshold which lines to display:' % (len(grades), ''.join('%s ' % c for c in grades)))
if threshold > -1:
for line in trace:
if line.grade >= threshold:
print line.grade, line.to_str_line()
except Exception, e:
w.close()
msg(e.message + '\n')
def PopulateModel(self):
self.Clean()
vmr = get_vmr()
w = NotifyProgress()
w.show()
ctr = 0
max = len(self.trace)
# present clustering analysis in viewer
prev_ctx = defaultdict(lambda: 0)
for line in self.trace:
ctr += 1
w.pbar_set(int(float(ctr) / float(max) * 100))
if isinstance(line, Traceline):
tid = QtGui.QStandardItem('%s' % line.thread_id)
addr = QtGui.QStandardItem('%x' % line.addr)
disasm = QtGui.QStandardItem(line.disasm_str())
comment = QtGui.QStandardItem(''.join(c for c in line.comment if line.comment is not None))
context = QtGui.QStandardItem(''.join('%s:%s ' % (c, line.ctx[c]) for c in line.ctx if line.ctx is not None))
prev_ctx = line.ctx
self.sim.appendRow([tid, addr, disasm, comment, context])
else:
cluster_node = QtGui.QStandardItem('Cluster %x-%x' % (line[0].addr, line[-1].addr))
self.sim.appendRow(cluster_node)
if vmr.bb:
cluster = line
bbs = []
bb = []
# subdivide the clusters by basic blocks
for line in cluster:
assert isinstance(line, Traceline)
if line.disasm[0].startswith('j'):
bb.append(line)
bbs.append(bb)
bb = []
else:
bb.append(line)
for bb in bbs:
bb_sum = self.bb_func(bb, self.ctx_reg_size, prev_ctx)
bb_node = QtGui.QStandardItem(
'BB%s Summary %x-%x: %s\t%s\t%s' % (bbs.index(bb), bb[0].addr, bb[-1].addr,
''.join('%s ; ' % (''.join('%s, ' % c for c in line)) for line in bb_sum.disasm),
''.join('%s, ' % c for c in filter(None, bb_sum.comment) if bb_sum.comment is not None),
''.join('%s:%s ' % (c, bb_sum.ctx[c]) for c in bb_sum.ctx if bb_sum.ctx is not None)))
for line in bb:
tid = QtGui.QStandardItem('%s' % line.thread_id)
addr = QtGui.QStandardItem('%x' % line.addr)
disasm = QtGui.QStandardItem(line.disasm_str())
comment = QtGui.QStandardItem(''.join(c for c in line.comment if line.comment is not None))
context = QtGui.QStandardItem(
''.join('%s:%s ' % (c, line.ctx[c]) for c in line.ctx if line.ctx is not None))
bb_node.appendRow([tid, addr, disasm, comment, context])
cluster_node.appendRow(bb_node)
self.treeView.setFirstColumnSpanned(bbs.index(bb), cluster_node.index(), True)
prev_ctx = bb[-1].ctx
else:
for l in line:
tid = QtGui.QStandardItem('%s' % l.thread_id)
addr = QtGui.QStandardItem('%x' % l.addr)
disasm = QtGui.QStandardItem(l.disasm_str())
comment = QtGui.QStandardItem(''.join(c for c in l.comment if l.comment is not None))
context = QtGui.QStandardItem(''.join('%s:%s ' % (c, l.ctx[c]) for c in l.ctx if l.ctx is not None))
cluster_node.appendRow([tid, addr, disasm, comment, context])
w.close()
self.treeView.resizeColumnToContents(0)
self.treeView.resizeColumnToContents(1)
self.treeView.resizeColumnToContents(2)
self.treeView.resizeColumnToContents(3)
self.treeView.resizeColumnToContents(4)
def prepare_vm_operands():
vmr = get_vmr()
return deepcopy(vmr.vm_operands)
def load_trace():
vmr = get_vmr()
trace = load()
vmr.trace = trace
def prepare_trace():
vmr = get_vmr()
if vmr.trace is None:
vmr.trace = load()
return deepcopy(vmr.trace)
base_addr,
'Could not determine the Startaddr of the jmp table, please specify: '
)
code_start = PrevAddr(vm_seg_end)
while not GetDisasm(code_start).__contains__('jmp'):
code_start = PrevAddr(code_start)
code_end = vm_seg_end
code_start += 1
vm_ctx.code_start = code_start
vm_ctx.code_end = code_end
vm_ctx.base_addr = base_addr
vm_ctx.vm_addr = vm_addr
vmr = get_vmr()
vmr.vm_ctx = vm_ctx
if manual:
print 'Code Start: %x; Code End: %x; Base Addr: %x; VM Addr: %x' % (
code_start, code_end, base_addr, vm_addr)
def static_deobfuscate(display=0, userchoice=False):
"""
Wrapper for deobfuscate function which allows for manual user Input.
:param display: Bool -> use output window or BBGraphViewer
:param userchoice: let user input vm context values
"""
vmr = get_vmr()
if vmr.code_start == BADADDR:
def load_trace():
vmr = get_vmr()
trace = load()
vmr.trace = trace
def prepare_vm_ctx():
vmr = get_vmr()
return deepcopy(vmr.vm_ctx)
def prepare_vm_operands():
vmr = get_vmr()
return deepcopy(vmr.vm_operands)
def dbg_trace(self, tid, ea):
"""
处理trace事件回调
:param tid:
:param ea:
:return:
"""
vmr = get_vmr()
try:
if vmr.extract_param and GetDisasm(ea).__contains__('call'):
run_var = 0
key = GetDisasm(ea).split('call')[1].strip()
while True:
# traverse trace backwards and get sequential push and mov params
line = self.trace[-(run_var + 1)]
if line.is_push and line.disasm_len == 2:
try:
self.func_args[key].add(line.ctx[get_reg(
line.disasm[1], self.arch)])
except:
self.func_args[key].add(line.disasm[1])
elif line.is_mov:
try:
self.func_args[key].add(line.ctx[get_reg(
line.disasm[2], self.arch)])
except:
self.func_args[key].add(line.disasm[2])
else:
break
run_var += 1
# TODO mmx xmmx ymmx
# compute next ctx
if self.arch == 32:
self.ctx = defaultdict(
lambda: '0', {
'eax': self.convert(cpu.eax),
'ebx': self.convert(cpu.ebx),
'edx': self.convert(cpu.edx),
'ecx': self.convert(cpu.ecx),
'ebp': self.convert(cpu.ebp),
'esp': self.convert(cpu.esp),
'eip': self.convert(cpu.eip),
'edi': self.convert(cpu.edi),
'esi': self.convert(cpu.esi),
'cf': self.convert(cpu.cf),
'zf': self.convert(cpu.zf),
'sf': self.convert(cpu.sf),
'of': self.convert(cpu.of),
'pf': self.convert(cpu.pf),
'af': self.convert(cpu.af),
'tf': self.convert(cpu.tf),
'df': self.convert(cpu.df)
})
elif self.arch == 64:
self.ctx = defaultdict(
lambda: '0', {
'rax': self.convert(cpu.eax),
'rbx': self.convert(cpu.ebx),
'rdx': self.convert(cpu.edx),
'rcx': self.convert(cpu.ecx),
'rbp': self.convert(cpu.ebp),
'rsp': self.convert(cpu.esp),
'rip': self.convert(cpu.eip),
'edi': self.convert(cpu.edi),
'rsi': self.convert(cpu.rsi),
'r8': self.convert(cpu.r8),
'r9': self.convert(cpu.r9),
'r10': self.convert(cpu.r10),
'r11': self.convert(cpu.r11),
'r12': self.convert(cpu.r12),
'r13': self.convert(cpu.r13),
'r14': self.convert(cpu.r14),
'r15': self.convert(cpu.r15),
'cf': self.convert(cpu.cf),
'zf': self.convert(cpu.zf),
'sf': self.convert(cpu.sf),
'of': self.convert(cpu.of),
'pf': self.convert(cpu.pf),
'af': self.convert(cpu.af),
'tf': self.convert(cpu.tf),
'df': self.convert(cpu.df)
})
self.trace.append(
Traceline(thread_id=tid,
addr=ea,
disasm=self.disconv(GetDisasm(ea)),
ctx=deepcopy(self.ctx)))
except Exception, e:
print e.message
def PopulateModel(self):
self.Clean()
vmr = get_vmr()
w = NotifyProgress()
w.show()
ctr = 0
max = len(self.trace)
# present clustering analysis in viewer
prev_ctx = defaultdict(lambda: 0)
for line in self.trace:
ctr += 1
w.pbar_set(int(float(ctr) / float(max) * 100))
if isinstance(line, Traceline):
tid = QtGui.QStandardItem('%s' % line.thread_id)
addr = QtGui.QStandardItem('%x' % line.addr)
disasm = QtGui.QStandardItem(line.disasm_str())
comment = QtGui.QStandardItem(''.join(
c for c in line.comment if line.comment is not None))
context = QtGui.QStandardItem(''.join(
'%s:%s ' % (c, line.ctx[c]) for c in line.ctx
if line.ctx is not None))
prev_ctx = line.ctx
self.sim.appendRow([tid, addr, disasm, comment, context])
else:
cluster_node = QtGui.QStandardItem(
'Cluster %x-%x' % (line[0].addr, line[-1].addr))
self.sim.appendRow(cluster_node)
if vmr.bb:
cluster = line
bbs = []
bb = []
# subdivide the clusters by basic blocks
for line in cluster:
assert isinstance(line, Traceline)
if is_basic_block_end(line.addr):
bb.append(line)
bbs.append(bb)
bb = []
else:
bb.append(line)
for bb in bbs:
bb_sum = self.bb_func(bb, self.ctx_reg_size, prev_ctx)
bb_node = QtGui.QStandardItem(
'BB%s Summary %x-%x: %s\t%s\t%s' %
(bbs.index(bb), bb[0].addr, bb[-1].addr, ''.join(
'%s ; ' % (''.join('%s, ' % c for c in line))
for line in bb_sum.disasm), ''.join(
'%s, ' % c
for c in filter(None, bb_sum.comment)
if bb_sum.comment is not None), ''.join(
'%s:%s ' % (c, bb_sum.ctx[c])
for c in bb_sum.ctx
if bb_sum.ctx is not None)))
for line in bb:
tid = QtGui.QStandardItem('%s' % line.thread_id)
addr = QtGui.QStandardItem('%x' % line.addr)
disasm = QtGui.QStandardItem(line.disasm_str())
comment = QtGui.QStandardItem(''.join(
c for c in line.comment
if line.comment is not None))
context = QtGui.QStandardItem(''.join(
'%s:%s ' % (c, line.ctx[c]) for c in line.ctx
if line.ctx is not None))
bb_node.appendRow(
[tid, addr, disasm, comment, context])
cluster_node.appendRow(bb_node)
self.treeView.setFirstColumnSpanned(
bbs.index(bb), cluster_node.index(), True)
prev_ctx = bb[-1].ctx
else:
for l in line:
tid = QtGui.QStandardItem('%s' % l.thread_id)
addr = QtGui.QStandardItem('%x' % l.addr)
disasm = QtGui.QStandardItem(l.disasm_str())
comment = QtGui.QStandardItem(''.join(
c for c in l.comment if l.comment is not None))
context = QtGui.QStandardItem(''.join(
'%s:%s ' % (c, l.ctx[c]) for c in l.ctx
if l.ctx is not None))
cluster_node.appendRow(
[tid, addr, disasm, comment, context])
w.close()
self.treeView.resizeColumnToContents(0)
self.treeView.resizeColumnToContents(1)
self.treeView.resizeColumnToContents(2)
self.treeView.resizeColumnToContents(3)
self.treeView.resizeColumnToContents(4)
def prepare_vm_ctx():
vmr = get_vmr()
return deepcopy(vmr.vm_ctx)
