def pre_run(isClustering, testfile):
# check version of python running
if not sys.version_info[:2] == (2, 7):
u.vd_error("U need python 2.7 installed to run Virtual Deobfuscator")
print "\nLoading packages..."
# clean up Virtual Deob database and testing file
if isClustering == False:
if os.path.exists(DB):
os.remove(DB)
if os.path.exists(testfile):
os.remove(testfile)
try:
# use lxml if it's available -- faster
from lxml import etree
print("- running with lxml.etree")
except ImportError:
# Python 2.5
import xml.etree.cElementTree as etree
print("- running with cElementTree on Python 2.5+")
return etree
def pre_run(isClustering, testfile):
# check version of python running
if not sys.version_info[:2] == (2, 7):
u.vd_error("U need python 2.7 installed to run Virtual Deobfuscator")
print "\nLoading packages..."
# clean up Virtual Deob database and testing file
if isClustering == False:
if os.path.exists(DB):
os.remove(DB)
if os.path.exists(testfile):
os.remove(testfile)
try:
# use lxml if it's available -- faster
from lxml import etree
print("- running with lxml.etree")
except ImportError:
# Python 2.5
import xml.etree.cElementTree as etree
print("- running with cElementTree on Python 2.5+")
return etree
def create_IR(runtrace):
if os.path.exists(IR_FILE):
os.remove(IR_FILE)
print ""
sys.stdout.write("formating runtrace for Virtual Deobfuscator > %s " \
% IR_FILE)
try:
ofile = open(IR_FILE, "a")
except IOError as exc:
raise IOError("%s: %s" % (IR_FILE, exc.strerror))
# verify parameters are correct
if not os.path.exists(runtrace):
u.vd_error("No such runtrace file: " + runtrace)
try:
with open(runtrace, "r") as ifile:
cnt = 0
while 1:
line = ifile.readline()
if len(line) == 0:
break
line = line.strip("\n")
# remove special chars from runtrace
line = re.sub(r'[\<\>\&]', " ", line)
line = filter(lambda x: x in string.printable, line)
ofile.writelines(line)
ofile.writelines("\n")
cnt += 1
# basic progress bar
if cnt > 5000:
sys.stdout.write(".")
sys.stdout.flush()
cnt = 0
ifile.closed
ofile.closed
except IOError as exc:
raise IOError("%s: %s" % (runtrace, exc.strerror))
sys.exit()
def create_IR(runtrace):
if os.path.exists(IR_FILE):
os.remove(IR_FILE)
print ""
sys.stdout.write("formating runtrace for Virtual Deobfuscator > %s " \
% IR_FILE)
try:
ofile = open(IR_FILE, "a")
except IOError as exc:
raise IOError("%s: %s" % (IR_FILE, exc.strerror))
# verify parameters are correct
if not os.path.exists(runtrace):
u.vd_error("No such runtrace file: " + runtrace)
try:
with open(runtrace, "r") as ifile:
cnt = 0
while 1:
line = ifile.readline()
if len(line) == 0:
break
line = line.strip("\n")
# remove special chars from runtrace
line = re.sub(r'[\<\>\&]', " ", line)
line = filter(lambda x: x in string.printable, line)
ofile.writelines(line)
ofile.writelines("\n")
cnt += 1
# basic progress bar
if cnt > 5000:
sys.stdout.write(".")
sys.stdout.flush()
cnt = 0
ifile.closed
ofile.closed
except IOError as exc:
raise IOError("%s: %s" % (runtrace, exc.strerror))
sys.exit()
def pre_run(isClustering, testfile):
# check version of python running
if not sys.version_info[:2] == (2, 7):
u.vd_error("U need python 2.7 installed to run Virtual Deobfuscator")
print "\nLoading packages..."
# clean up Virtual Deob database and testing file
if isClustering == False:
if os.path.exists(DB):
os.remove(DB)
if os.path.exists(testfile):
os.remove(testfile)
# use lxml if it's available -- faster
from lxml import etree
return etree
def pre_run(isClustering, testfile):
# check version of python running
if not sys.version_info[:2] == (2, 7):
u.vd_error("U need python 2.7 installed to run Virtual Deobfuscator")
print "\nLoading packages..."
# clean up Virtual Deob database and testing file
if isClustering == False:
if os.path.exists(DB):
os.remove(DB)
if os.path.exists(testfile):
os.remove(testfile)
# use lxml if it's available -- faster
from lxml import etree
return etree
def main():
print "\n------------------------------------------------------------------"
print "| Virtual Deobfuscator ver 1.0 |"
print "| HexEffect |"
print "------------------------------------------------------------------\n"
run_clustering = False
cleanup_rt = False # clean up runtrace of ill formated lines
unfolding = False
build_sections = False
section = 0
dbgr = -1
#--------------------------------------------------------------------------
# Handle command line arguments
testXML = ''
epilog = '''
------------------------------------------------------------------
Example:
Import a trace with:
VirtualDeobfuscator.py -i trace.txt -d 1 -t testXML.txt
Output will be:
vd.xml VD database
vd_IR.txt just the runtrace with special chars removed
testXML.txt file that is used to test the XML parsing
Compare testXML.txt and vd_IR.txt:
diff testXML.txt vd_IR.txt
'''
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=epilog)
import_args = parser.add_argument_group(
'Import', 'These arguments are used to import a trace file.')
import_args.add_argument(
'-i', '--ifile', help='A runtrace file created by a debugger')
import_args.add_argument(
'-d', '--debugger', help='Debugger: 1 = Olly 2.0, 2 = Immunity/Olly '
'1.0, 3 = WinDbg', type=int, choices=(1, 2, 3))
import_args.add_argument(
'-t', '--testXML', help='A test file that can be used to verify trace '
'file conversion to XML', default='')
import_args.add_argument(
'-x', '--cleanup', action='store_true', default=False,
help='Cleanup the runtrace')
cluster_args = parser.add_argument_group(
'Clustering', 'These arguments are used to perform clustering on an '
'imported trace.')
cluster_args.add_argument(
'-c', '--cluster', action='store_true', default=False,
help='Run clustering on a database')
cluster_args.add_argument(
'-u', '--unfold', action='store_true', default=False,
help='Unfold clusters')
cluster_args.add_argument(
'-s', '--size', type=int, help='Decompose sections of clusters '
'according to this filter size')
args = parser.parse_args()
if not (args.ifile or args.cluster):
parser.error('''You must specify a trace file (with -i) or
clustering mode (-c).''')
runtrace = args.ifile
testXML = args.testXML
cleanup_rt = args.cleanup
dbgr = args.debugger
run_clustering = args.cluster
unfolding = args.unfold
section = args.size
build_sections = args.size is not None
#----------------------------------------------------------------------
# print out header info
etree = hdr(run_clustering, dbgr, testXML)
#--------------------------------------------------------------------------
# verify parameters are correct
if run_clustering == False:
if not os.path.exists(runtrace):
u.vd_error("No such runtrace file: " + runtrace)
if dbgr > 3:
u.vd_error("Unknown <debugger>: supported OllyDbg 2.0 = 1, " \
"Immunity = 2, WinDbg = 3")
#----------------------------------------------------------------------
# generate intermediate representation file of runtrace...used for
# testing XML parsing and reading
if testXML != "" and (dbgr == u.OLLY or dbgr == u.IMMUNITY):
create_IR(runtrace)
if build_sections and section <=0:
u.vd_error("Cluster section filter must be greater than zero")
try:
db_file = open(DB, "a")
except IOError as exc:
raise IOError("%s: %s" % (DB, exc.strerror))
db_file.writelines("<root>\n")
#--------------------------------------------------------------------------
# clean up bad lines in runtrace
if cleanup_rt:
reformat_rt(runtrace)
#--------------------------------------------------------------------------
# parse runtrace
if run_clustering == False:
with open(runtrace, "r") as ifile:
line_cnt = 0
cnt = 0
line_cnt = 0
while 1:
line_cnt += 1
line = ifile.readline()
line_len = len(line)
if line_len == 0:
break
line = line.strip("\n")
#--------------------------------------------------------------
# OllyDbg 2.0
if dbgr == u.OLLY:
# check for end of runtrace
if line == "-------- End of session" or \
line == "-------- Logging stopped":
break
if line_len < 38:
print "\nIll formed line: ", line, line_cnt
print "Use option -x to remove bad lines from runtrace"
u.vd_error("PLEASE Fix line to continue")
if line_cnt == 1:
tmp_lst = line.strip().split()
# the first string should be <thread> i.e. 'main'
if tmp_lst[0] == "Address":
u.vd_error("Incorrect runtrace format for Olly")
parse_OllyDbg_20(db_file, line, etree)
#--------------------------------------------------------------
# Immunity/OllyDbg 1.10 debugger
elif dbgr == u.IMMUNITY:
# check for end of runtrace
if line == " Run trace closed" or \
line.find("Process terminated") != -1:
break
if line_len < 21:
print "\nSkipping ill formed line: ", line, line_cnt
u.vd_error("PLEASE Fix line to continue")
# Skip first line for Immunity runtrace
# Address Thread Command ; Registers and comments
if line_cnt == 1:
# Verify if correct format for Immunity runtrace
if line.find('Address') == -1:
u.vd_error("Incorrect runtrace format for Immunity")
continue
parse_Immunity_Olly110(db_file, line, etree)
#--------------------------------------------------------------
# WinDBG
elif dbgr == u.WINDBG:
# check for end of runtrace
if line.find("logclose") != -1:
break
if line_len < 45:
print "\nSkipping ill formed line: ", line, line_cnt
u.vd_error("PLEASE Fix line to continue")
# Skip two lines for WinDbg runtrace
# Opened log file 'C:path\...\wd_hello_o.txt'
# 0:000> pa 40118B
if line_cnt < 2:
continue
parse_WinDbg(db_file, line, etree)
else:
u.vd_error("Make sure to declare a debugger -d switch")
cnt += 1
if cnt > 5000:
sys.stdout.write(".")
sys.stdout.flush
cnt = 0
ifile.closed
# tack on </root> endo of xml doc
db_file.writelines("</root>\n")
db_file.close()
#just for testing, parse back in and write to a temp file to diff
read_xml(etree, testXML, dbgr, cluster)
footer(line_cnt)
#--------------------------------------------------------------------------
# Cluster
if run_clustering == True:
if not os.path.exists(DB):
u.vd_error("No such database file: " + DB)
#------------------------------------------------------------------
# No files read in...already have the xml file
#------------------------------------------------------------------
read_xml(etree, testXML, dbgr, cluster)
cluster.print_cluster(-1, False)
round = 0
cluster.build_freq_graph(round)
cluster.print_freq_graph(round, False)
cluster.compress_basic_blocks_loop(round)
cluster.print_compress_window(round, False)
cluster.print_compress_backtrace(round, False)
cluster.compress_create_new_cluster(round)
cluster.compress_backtrace(round, False)
cluster.cluster_lst = cluster.new_cluster_lst[:]
cluster.print_cluster(round, False)
round = 1
cluster.build_freq_graph(round)
cluster.print_freq_graph(round, False)
while 1:
print ("\nGreedy round %s:------------------------------" \
% u.id_round[round])
greedy = cluster.greedy_new_cluster(round)
if greedy and round < 26:
cluster.cluster_lst = cluster.new_cluster_lst[:]
cluster.print_cluster(round, False)
cluster.print_greedy_backtrace(round, False)
cluster.greedy_backtrace(round, False)
build_complete_backtrace(round, cluster)
# UNFOLDING
if unfolding:
unfold = cluster.unfold_single_clusters(round)
if unfold:
cluster.cluster_lst = cluster.new_cluster_lst[:]
round += 1
cluster.build_freq_graph(round)
cluster.print_freq_graph(round, False)
else:
# temp
cluster.final_assembly(round-1, etree, dbgr)
cluster.chunking_clusters(etree, dbgr)
if build_sections:
cluster.chunking_sections(etree, dbgr, section)
print ("\nClustering done\n")
sys.exit(2)
def main():
print "\n------------------------------------------------------------------"
print "| Virtual Deobfuscator ver 1.0 |"
print "| HexEffect |"
print "------------------------------------------------------------------\n"
run_clustering = False
cleanup_rt = False # clean up runtrace of ill formated lines
unfolding = False
build_sections = False
section = 0
dbgr = -1
#--------------------------------------------------------------------------
# Handle command line arguments
testXML = ''
epilog = '''
------------------------------------------------------------------
Example:
Import a trace with:
VirtualDeobfuscator.py -i trace.txt -d 1 -t testXML.txt
Output will be:
vd.xml VD database
vd_IR.txt just the runtrace with special chars removed
testXML.txt file that is used to test the XML parsing
Compare testXML.txt and vd_IR.txt:
diff testXML.txt vd_IR.txt
'''
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter, epilog=epilog)
import_args = parser.add_argument_group(
'Import', 'These arguments are used to import a trace file.')
import_args.add_argument('-i',
'--ifile',
help='A runtrace file created by a debugger')
import_args.add_argument('-d',
'--debugger',
help='Debugger: 1 = Olly 2.0, 2 = Immunity/Olly '
'1.0, 3 = WinDbg',
type=int,
choices=(1, 2, 3))
import_args.add_argument(
'-t',
'--testXML',
help='A test file that can be used to verify trace '
'file conversion to XML',
default='')
import_args.add_argument('-x',
'--cleanup',
action='store_true',
default=False,
help='Cleanup the runtrace')
cluster_args = parser.add_argument_group(
'Clustering', 'These arguments are used to perform clustering on an '
'imported trace.')
cluster_args.add_argument('-c',
'--cluster',
action='store_true',
default=False,
help='Run clustering on a database')
cluster_args.add_argument('-u',
'--unfold',
action='store_true',
default=False,
help='Unfold clusters')
cluster_args.add_argument('-s',
'--size',
type=int,
help='Decompose sections of clusters '
'according to this filter size')
args = parser.parse_args()
if not (args.ifile or args.cluster):
parser.error('''You must specify a trace file (with -i) or
clustering mode (-c).''')
runtrace = args.ifile
testXML = args.testXML
cleanup_rt = args.cleanup
dbgr = args.debugger
run_clustering = args.cluster
unfolding = args.unfold
section = args.size
build_sections = args.size is not None
#----------------------------------------------------------------------
# print out header info
etree = hdr(run_clustering, dbgr, testXML)
#--------------------------------------------------------------------------
# verify parameters are correct
if run_clustering == False:
if not os.path.exists(runtrace):
u.vd_error("No such runtrace file: " + runtrace)
if dbgr > 3:
u.vd_error("Unknown <debugger>: supported OllyDbg 2.0 = 1, " \
"Immunity = 2, WinDbg = 3")
#----------------------------------------------------------------------
# generate intermediate representation file of runtrace...used for
# testing XML parsing and reading
if testXML != "" and (dbgr == u.OLLY or dbgr == u.IMMUNITY):
create_IR(runtrace)
if build_sections and section <= 0:
u.vd_error("Cluster section filter must be greater than zero")
try:
db_file = open(DB, "a")
except IOError as exc:
raise IOError("%s: %s" % (DB, exc.strerror))
db_file.writelines("<root>\n")
#--------------------------------------------------------------------------
# clean up bad lines in runtrace
if cleanup_rt:
reformat_rt(runtrace)
#--------------------------------------------------------------------------
# parse runtrace
if run_clustering == False:
with open(runtrace, "r") as ifile:
line_cnt = 0
cnt = 0
line_cnt = 0
while 1:
line_cnt += 1
line = ifile.readline()
line_len = len(line)
if line_len == 0:
break
line = line.strip("\n")
#--------------------------------------------------------------
# OllyDbg 2.0
if dbgr == u.OLLY:
# check for end of runtrace
if line == "-------- End of session" or \
line == "-------- Logging stopped":
break
if line_len < 38:
print "\nIll formed line: ", line, line_cnt
print "Use option -x to remove bad lines from runtrace"
u.vd_error("PLEASE Fix line to continue")
if line_cnt == 1:
tmp_lst = line.strip().split()
# the first string should be <thread> i.e. 'main'
if tmp_lst[0] == "Address":
u.vd_error("Incorrect runtrace format for Olly")
parse_OllyDbg_20(db_file, line, etree)
#--------------------------------------------------------------
# Immunity/OllyDbg 1.10 debugger
elif dbgr == u.IMMUNITY:
# check for end of runtrace
if line == " Run trace closed" or \
line.find("Process terminated") != -1:
break
if line_len < 21:
print "\nSkipping ill formed line: ", line, line_cnt
u.vd_error("PLEASE Fix line to continue")
# Skip first line for Immunity runtrace
# Address Thread Command ; Registers and comments
if line_cnt == 1:
# Verify if correct format for Immunity runtrace
if line.find('Address') == -1:
u.vd_error(
"Incorrect runtrace format for Immunity")
continue
parse_Immunity_Olly110(db_file, line, etree)
#--------------------------------------------------------------
# WinDBG
elif dbgr == u.WINDBG:
# check for end of runtrace
if line.find("logclose") != -1:
break
if line_len < 45:
print "\nSkipping ill formed line: ", line, line_cnt
u.vd_error("PLEASE Fix line to continue")
# Skip two lines for WinDbg runtrace
# Opened log file 'C:path\...\wd_hello_o.txt'
# 0:000> pa 40118B
if line_cnt < 2:
continue
parse_WinDbg(db_file, line, etree)
else:
u.vd_error("Make sure to declare a debugger -d switch")
cnt += 1
if cnt > 5000:
sys.stdout.write(".")
sys.stdout.flush
cnt = 0
ifile.closed
# tack on </root> endo of xml doc
db_file.writelines("</root>\n")
db_file.close()
#just for testing, parse back in and write to a temp file to diff
read_xml(etree, testXML, dbgr, cluster)
footer(line_cnt)
#--------------------------------------------------------------------------
# Cluster
if run_clustering == True:
if not os.path.exists(DB):
u.vd_error("No such database file: " + DB)
#------------------------------------------------------------------
# No files read in...already have the xml file
#------------------------------------------------------------------
read_xml(etree, testXML, dbgr, cluster)
cluster.print_cluster(-1, False)
round = 0
cluster.build_freq_graph(round)
cluster.print_freq_graph(round, False)
cluster.compress_basic_blocks_loop(round)
cluster.print_compress_window(round, False)
cluster.print_compress_backtrace(round, False)
cluster.compress_create_new_cluster(round)
cluster.compress_backtrace(round, False)
cluster.cluster_lst = cluster.new_cluster_lst[:]
cluster.print_cluster(round, False)
round = 1
cluster.build_freq_graph(round)
cluster.print_freq_graph(round, False)
while 1:
print ("\nGreedy round %s:------------------------------" \
% u.id_round[round])
greedy = cluster.greedy_new_cluster(round)
if greedy and round < 26:
cluster.cluster_lst = cluster.new_cluster_lst[:]
cluster.print_cluster(round, False)
cluster.print_greedy_backtrace(round, False)
cluster.greedy_backtrace(round, False)
build_complete_backtrace(round, cluster)
# UNFOLDING
if unfolding:
unfold = cluster.unfold_single_clusters(round)
if unfold:
cluster.cluster_lst = cluster.new_cluster_lst[:]
round += 1
cluster.build_freq_graph(round)
cluster.print_freq_graph(round, False)
else:
# temp
cluster.final_assembly(round - 1, etree, dbgr)
cluster.chunking_clusters(etree, dbgr)
if build_sections:
cluster.chunking_sections(etree, dbgr, section)
print("\nClustering done\n")
sys.exit(2)