예제 #1
0
def pre_run(isClustering, testfile):

    # check version of python running
    if not sys.version_info[:2] == (2, 7):
        u.vd_error("U need python 2.7 installed to run Virtual Deobfuscator")

    print "\nLoading packages..."

    # clean up Virtual Deob database and testing file
    if isClustering == False:
        if os.path.exists(DB):
            os.remove(DB)
        if os.path.exists(testfile):
            os.remove(testfile)

    try:
        # use lxml if it's available -- faster
        from lxml import etree
        print("- running with lxml.etree")
    except ImportError:
        # Python 2.5
        import xml.etree.cElementTree as etree
        print("- running with cElementTree on Python 2.5+")

    return etree
def pre_run(isClustering, testfile):

    # check version of python running
    if not sys.version_info[:2] == (2, 7):
        u.vd_error("U need python 2.7 installed to run Virtual Deobfuscator")

    print "\nLoading packages..."

    # clean up Virtual Deob database and testing file
    if isClustering == False:
        if os.path.exists(DB):
            os.remove(DB)
        if os.path.exists(testfile):
            os.remove(testfile)

    try:
        # use lxml if it's available -- faster
        from lxml import etree
        print("- running with lxml.etree")
    except ImportError:
        # Python 2.5
        import xml.etree.cElementTree as etree
        print("- running with cElementTree on Python 2.5+")

    return etree
def create_IR(runtrace):

    if os.path.exists(IR_FILE):
        os.remove(IR_FILE)

    print ""
    sys.stdout.write("formating runtrace for Virtual Deobfuscator > %s " \
                     % IR_FILE)


    try:
        ofile = open(IR_FILE, "a")
    except IOError as exc:
        raise IOError("%s: %s" % (IR_FILE, exc.strerror))

    # verify parameters are correct
    if not os.path.exists(runtrace):
        u.vd_error("No such runtrace file: " + runtrace)

    try:
        with open(runtrace, "r") as ifile:
            cnt = 0
            while 1:
                line = ifile.readline()
                if len(line) == 0:
                    break

                line = line.strip("\n")

                # remove special chars from runtrace
                line = re.sub(r'[\<\>\&]', " ", line)
                line = filter(lambda x: x in string.printable, line)

                ofile.writelines(line)
                ofile.writelines("\n")
                cnt += 1
                # basic progress bar
                if cnt > 5000:
                    sys.stdout.write(".")
                    sys.stdout.flush()
                    cnt = 0

            ifile.closed
            ofile.closed

    except IOError as exc:
        raise IOError("%s: %s" % (runtrace, exc.strerror))
        sys.exit()
예제 #4
0
def create_IR(runtrace):

    if os.path.exists(IR_FILE):
        os.remove(IR_FILE)

    print ""
    sys.stdout.write("formating runtrace for Virtual Deobfuscator > %s " \
                     % IR_FILE)


    try:
        ofile = open(IR_FILE, "a")
    except IOError as exc:
        raise IOError("%s: %s" % (IR_FILE, exc.strerror))

    # verify parameters are correct
    if not os.path.exists(runtrace):
        u.vd_error("No such runtrace file: " + runtrace)

    try:
        with open(runtrace, "r") as ifile:
            cnt = 0
            while 1:
                line = ifile.readline()
                if len(line) == 0:
                    break

                line = line.strip("\n")

                # remove special chars from runtrace
                line = re.sub(r'[\<\>\&]', " ", line)
                line = filter(lambda x: x in string.printable, line)

                ofile.writelines(line)
                ofile.writelines("\n")
                cnt += 1
                # basic progress bar
                if cnt > 5000:
                    sys.stdout.write(".")
                    sys.stdout.flush()
                    cnt = 0

            ifile.closed
            ofile.closed

    except IOError as exc:
        raise IOError("%s: %s" % (runtrace, exc.strerror))
        sys.exit()
def pre_run(isClustering, testfile):

    # check version of python running
    if not sys.version_info[:2] == (2, 7):
        u.vd_error("U need python 2.7 installed to run Virtual Deobfuscator")

    print "\nLoading packages..."

    # clean up Virtual Deob database and testing file
    if isClustering == False:
        if os.path.exists(DB):
            os.remove(DB)
        if os.path.exists(testfile):
            os.remove(testfile)

    # use lxml if it's available -- faster
    from lxml import etree
    return etree
def pre_run(isClustering, testfile):

    # check version of python running
    if not sys.version_info[:2] == (2, 7):
        u.vd_error("U need python 2.7 installed to run Virtual Deobfuscator")

    print "\nLoading packages..."

    # clean up Virtual Deob database and testing file
    if isClustering == False:
        if os.path.exists(DB):
            os.remove(DB)
        if os.path.exists(testfile):
            os.remove(testfile)

    # use lxml if it's available -- faster
    from lxml import etree
    return etree
def main():

    print "\n------------------------------------------------------------------"
    print "|                  Virtual Deobfuscator ver 1.0                  |"
    print "|                            HexEffect                           |"
    print "------------------------------------------------------------------\n"

    run_clustering = False
    cleanup_rt     = False  # clean up runtrace of ill formated lines
    unfolding      = False
    build_sections = False
    section        = 0
    dbgr           = -1

    #--------------------------------------------------------------------------
    # Handle command line arguments
    testXML = ''

    epilog = '''
------------------------------------------------------------------

Example:
  Import a trace with:

      VirtualDeobfuscator.py -i trace.txt -d 1 -t testXML.txt

  Output will be:

      vd.xml       VD database
      vd_IR.txt    just the runtrace with special chars removed
      testXML.txt  file that is used to test the XML parsing

  Compare testXML.txt and vd_IR.txt:

      diff testXML.txt vd_IR.txt
'''

    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog=epilog)
    import_args = parser.add_argument_group(
        'Import', 'These arguments are used to import a trace file.')
    import_args.add_argument(
        '-i', '--ifile', help='A runtrace file created by a debugger')
    import_args.add_argument(
        '-d', '--debugger', help='Debugger: 1 = Olly 2.0, 2 = Immunity/Olly '
        '1.0, 3 = WinDbg', type=int, choices=(1, 2, 3))
    import_args.add_argument(
        '-t', '--testXML', help='A test file that can be used to verify trace '
        'file conversion to XML', default='')
    import_args.add_argument(
        '-x', '--cleanup', action='store_true', default=False,
        help='Cleanup the runtrace')

    cluster_args = parser.add_argument_group(
        'Clustering', 'These arguments are used to perform clustering on an '
        'imported trace.')
    cluster_args.add_argument(
        '-c', '--cluster', action='store_true', default=False,
        help='Run clustering on a database')
    cluster_args.add_argument(
        '-u', '--unfold', action='store_true', default=False,
        help='Unfold clusters')
    cluster_args.add_argument(
        '-s', '--size', type=int, help='Decompose sections of clusters '
        'according to this filter size')
    args = parser.parse_args()

    if not (args.ifile or args.cluster):
        parser.error('''You must specify a trace file (with -i) or
clustering mode (-c).''')

    runtrace = args.ifile
    testXML = args.testXML
    cleanup_rt = args.cleanup
    dbgr = args.debugger
    run_clustering = args.cluster
    unfolding = args.unfold
    section = args.size
    build_sections = args.size is not None

    #----------------------------------------------------------------------
    # print out header info
    etree = hdr(run_clustering, dbgr, testXML)

    #--------------------------------------------------------------------------
    # verify parameters are correct
    if run_clustering == False:
        if not os.path.exists(runtrace):
            u.vd_error("No such runtrace file: " + runtrace)

        if dbgr > 3:
            u.vd_error("Unknown <debugger>: supported OllyDbg 2.0 = 1, " \
                     "Immunity = 2, WinDbg = 3")

        #----------------------------------------------------------------------
        # generate intermediate representation file of runtrace...used for
        # testing XML parsing and reading
        if testXML != "" and (dbgr == u.OLLY or dbgr == u.IMMUNITY):
            create_IR(runtrace)

        if build_sections and section <=0:
            u.vd_error("Cluster section filter must be greater than zero")

        try:
            db_file = open(DB, "a")
        except IOError as exc:
            raise IOError("%s: %s" % (DB, exc.strerror))

        db_file.writelines("<root>\n")

    #--------------------------------------------------------------------------
    # clean up bad lines in runtrace
    if cleanup_rt:
        reformat_rt(runtrace)

    #--------------------------------------------------------------------------
    # parse runtrace
    if run_clustering == False:

        with open(runtrace, "r") as ifile:

            line_cnt = 0
            cnt = 0
            line_cnt = 0

            while 1:
                line_cnt += 1
                line = ifile.readline()
                line_len = len(line)
                if line_len == 0:
                    break

                line = line.strip("\n")

                #--------------------------------------------------------------
                # OllyDbg 2.0
                if dbgr == u.OLLY:
                    # check for end of runtrace
                    if line == "--------  End of session" or \
                       line == "--------  Logging stopped":
                        break

                    if line_len < 38:
                        print "\nIll formed line: ", line, line_cnt
                        print "Use option -x to remove bad lines from runtrace"
                        u.vd_error("PLEASE Fix line to continue")

                    if line_cnt == 1:
                        tmp_lst = line.strip().split()
                        # the first string should be <thread> i.e. 'main'
                        if tmp_lst[0] == "Address":
                            u.vd_error("Incorrect runtrace format for Olly")
                    parse_OllyDbg_20(db_file, line, etree)
                #--------------------------------------------------------------
                # Immunity/OllyDbg 1.10 debugger
                elif dbgr == u.IMMUNITY:
                    # check for end of runtrace
                    if line == "    Run trace closed" or \
                       line.find("Process terminated") != -1:
                        break

                    if line_len < 21:
                        print "\nSkipping ill formed line: ", line, line_cnt
                        u.vd_error("PLEASE Fix line to continue")

                    # Skip first line for Immunity runtrace
                    # Address  Thread   Command  ; Registers and comments
                    if line_cnt == 1:
                        # Verify if correct format for Immunity runtrace
                        if line.find('Address') == -1:
                            u.vd_error("Incorrect runtrace format for Immunity")
                        continue
                    parse_Immunity_Olly110(db_file, line, etree)
                #--------------------------------------------------------------
                # WinDBG
                elif dbgr == u.WINDBG:
                    # check for end of runtrace
                    if line.find("logclose") != -1:
                        break

                    if line_len < 45:
                        print "\nSkipping ill formed line: ", line, line_cnt
                        u.vd_error("PLEASE Fix line to continue")

                    # Skip two lines for WinDbg runtrace
                    # Opened log file 'C:path\...\wd_hello_o.txt'
                    # 0:000> pa 40118B
                    if line_cnt < 2:
                        continue
                    parse_WinDbg(db_file, line, etree)
                else:
                    u.vd_error("Make sure to declare a debugger -d switch")

                cnt += 1
                if cnt > 5000:
                    sys.stdout.write(".")
                    sys.stdout.flush
                    cnt = 0

            ifile.closed

            # tack on </root> endo of xml doc
            db_file.writelines("</root>\n")
            db_file.close()

            #just for testing, parse back in and write to a temp file to diff
            read_xml(etree, testXML, dbgr, cluster)

            footer(line_cnt)

    #--------------------------------------------------------------------------
    # Cluster
    if run_clustering == True:
        if not os.path.exists(DB):
            u.vd_error("No such database file: " + DB)

        #------------------------------------------------------------------
        # No files read in...already have the xml file
        #------------------------------------------------------------------
        read_xml(etree, testXML, dbgr, cluster)

        cluster.print_cluster(-1, False)

        round = 0
        cluster.build_freq_graph(round)
        cluster.print_freq_graph(round, False)

        cluster.compress_basic_blocks_loop(round)
        cluster.print_compress_window(round, False)
        cluster.print_compress_backtrace(round, False)

        cluster.compress_create_new_cluster(round)
        cluster.compress_backtrace(round, False)

        cluster.cluster_lst = cluster.new_cluster_lst[:]
        cluster.print_cluster(round, False)

        round = 1

        cluster.build_freq_graph(round)
        cluster.print_freq_graph(round, False)

        while 1:
            print ("\nGreedy round %s:------------------------------" \
                    % u.id_round[round])
            greedy = cluster.greedy_new_cluster(round)
            if greedy and round < 26:
                cluster.cluster_lst = cluster.new_cluster_lst[:]
                cluster.print_cluster(round, False)

                cluster.print_greedy_backtrace(round, False)
                cluster.greedy_backtrace(round, False)
                build_complete_backtrace(round, cluster)

                # UNFOLDING
                if unfolding:
                    unfold = cluster.unfold_single_clusters(round)
                    if unfold:
                        cluster.cluster_lst = cluster.new_cluster_lst[:]

                round += 1
                cluster.build_freq_graph(round)
                cluster.print_freq_graph(round, False)

            else:
                # temp
                cluster.final_assembly(round-1, etree, dbgr)
                cluster.chunking_clusters(etree, dbgr)

                if build_sections:
                    cluster.chunking_sections(etree, dbgr, section)

                print ("\nClustering done\n")
                sys.exit(2)
예제 #8
0
def main():

    print "\n------------------------------------------------------------------"
    print "|                  Virtual Deobfuscator ver 1.0                  |"
    print "|                            HexEffect                           |"
    print "------------------------------------------------------------------\n"

    run_clustering = False
    cleanup_rt = False  # clean up runtrace of ill formated lines
    unfolding = False
    build_sections = False
    section = 0
    dbgr = -1

    #--------------------------------------------------------------------------
    # Handle command line arguments
    testXML = ''

    epilog = '''
------------------------------------------------------------------

Example:
  Import a trace with:

      VirtualDeobfuscator.py -i trace.txt -d 1 -t testXML.txt

  Output will be:

      vd.xml       VD database
      vd_IR.txt    just the runtrace with special chars removed
      testXML.txt  file that is used to test the XML parsing

  Compare testXML.txt and vd_IR.txt:

      diff testXML.txt vd_IR.txt
'''

    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter, epilog=epilog)
    import_args = parser.add_argument_group(
        'Import', 'These arguments are used to import a trace file.')
    import_args.add_argument('-i',
                             '--ifile',
                             help='A runtrace file created by a debugger')
    import_args.add_argument('-d',
                             '--debugger',
                             help='Debugger: 1 = Olly 2.0, 2 = Immunity/Olly '
                             '1.0, 3 = WinDbg',
                             type=int,
                             choices=(1, 2, 3))
    import_args.add_argument(
        '-t',
        '--testXML',
        help='A test file that can be used to verify trace '
        'file conversion to XML',
        default='')
    import_args.add_argument('-x',
                             '--cleanup',
                             action='store_true',
                             default=False,
                             help='Cleanup the runtrace')

    cluster_args = parser.add_argument_group(
        'Clustering', 'These arguments are used to perform clustering on an '
        'imported trace.')
    cluster_args.add_argument('-c',
                              '--cluster',
                              action='store_true',
                              default=False,
                              help='Run clustering on a database')
    cluster_args.add_argument('-u',
                              '--unfold',
                              action='store_true',
                              default=False,
                              help='Unfold clusters')
    cluster_args.add_argument('-s',
                              '--size',
                              type=int,
                              help='Decompose sections of clusters '
                              'according to this filter size')
    args = parser.parse_args()

    if not (args.ifile or args.cluster):
        parser.error('''You must specify a trace file (with -i) or
clustering mode (-c).''')

    runtrace = args.ifile
    testXML = args.testXML
    cleanup_rt = args.cleanup
    dbgr = args.debugger
    run_clustering = args.cluster
    unfolding = args.unfold
    section = args.size
    build_sections = args.size is not None

    #----------------------------------------------------------------------
    # print out header info
    etree = hdr(run_clustering, dbgr, testXML)

    #--------------------------------------------------------------------------
    # verify parameters are correct
    if run_clustering == False:
        if not os.path.exists(runtrace):
            u.vd_error("No such runtrace file: " + runtrace)

        if dbgr > 3:
            u.vd_error("Unknown <debugger>: supported OllyDbg 2.0 = 1, " \
                     "Immunity = 2, WinDbg = 3")

        #----------------------------------------------------------------------
        # generate intermediate representation file of runtrace...used for
        # testing XML parsing and reading
        if testXML != "" and (dbgr == u.OLLY or dbgr == u.IMMUNITY):
            create_IR(runtrace)

        if build_sections and section <= 0:
            u.vd_error("Cluster section filter must be greater than zero")

        try:
            db_file = open(DB, "a")
        except IOError as exc:
            raise IOError("%s: %s" % (DB, exc.strerror))

        db_file.writelines("<root>\n")

    #--------------------------------------------------------------------------
    # clean up bad lines in runtrace
    if cleanup_rt:
        reformat_rt(runtrace)

    #--------------------------------------------------------------------------
    # parse runtrace
    if run_clustering == False:

        with open(runtrace, "r") as ifile:

            line_cnt = 0
            cnt = 0
            line_cnt = 0

            while 1:
                line_cnt += 1
                line = ifile.readline()
                line_len = len(line)
                if line_len == 0:
                    break

                line = line.strip("\n")

                #--------------------------------------------------------------
                # OllyDbg 2.0
                if dbgr == u.OLLY:
                    # check for end of runtrace
                    if line == "--------  End of session" or \
                       line == "--------  Logging stopped":
                        break

                    if line_len < 38:
                        print "\nIll formed line: ", line, line_cnt
                        print "Use option -x to remove bad lines from runtrace"
                        u.vd_error("PLEASE Fix line to continue")

                    if line_cnt == 1:
                        tmp_lst = line.strip().split()
                        # the first string should be <thread> i.e. 'main'
                        if tmp_lst[0] == "Address":
                            u.vd_error("Incorrect runtrace format for Olly")
                    parse_OllyDbg_20(db_file, line, etree)
                #--------------------------------------------------------------
                # Immunity/OllyDbg 1.10 debugger
                elif dbgr == u.IMMUNITY:
                    # check for end of runtrace
                    if line == "    Run trace closed" or \
                       line.find("Process terminated") != -1:
                        break

                    if line_len < 21:
                        print "\nSkipping ill formed line: ", line, line_cnt
                        u.vd_error("PLEASE Fix line to continue")

                    # Skip first line for Immunity runtrace
                    # Address  Thread   Command  ; Registers and comments
                    if line_cnt == 1:
                        # Verify if correct format for Immunity runtrace
                        if line.find('Address') == -1:
                            u.vd_error(
                                "Incorrect runtrace format for Immunity")
                        continue
                    parse_Immunity_Olly110(db_file, line, etree)
                #--------------------------------------------------------------
                # WinDBG
                elif dbgr == u.WINDBG:
                    # check for end of runtrace
                    if line.find("logclose") != -1:
                        break

                    if line_len < 45:
                        print "\nSkipping ill formed line: ", line, line_cnt
                        u.vd_error("PLEASE Fix line to continue")

                    # Skip two lines for WinDbg runtrace
                    # Opened log file 'C:path\...\wd_hello_o.txt'
                    # 0:000> pa 40118B
                    if line_cnt < 2:
                        continue
                    parse_WinDbg(db_file, line, etree)
                else:
                    u.vd_error("Make sure to declare a debugger -d switch")

                cnt += 1
                if cnt > 5000:
                    sys.stdout.write(".")
                    sys.stdout.flush
                    cnt = 0

            ifile.closed

            # tack on </root> endo of xml doc
            db_file.writelines("</root>\n")
            db_file.close()

            #just for testing, parse back in and write to a temp file to diff
            read_xml(etree, testXML, dbgr, cluster)

            footer(line_cnt)

    #--------------------------------------------------------------------------
    # Cluster
    if run_clustering == True:
        if not os.path.exists(DB):
            u.vd_error("No such database file: " + DB)

        #------------------------------------------------------------------
        # No files read in...already have the xml file
        #------------------------------------------------------------------
        read_xml(etree, testXML, dbgr, cluster)

        cluster.print_cluster(-1, False)

        round = 0
        cluster.build_freq_graph(round)
        cluster.print_freq_graph(round, False)

        cluster.compress_basic_blocks_loop(round)
        cluster.print_compress_window(round, False)
        cluster.print_compress_backtrace(round, False)

        cluster.compress_create_new_cluster(round)
        cluster.compress_backtrace(round, False)

        cluster.cluster_lst = cluster.new_cluster_lst[:]
        cluster.print_cluster(round, False)

        round = 1

        cluster.build_freq_graph(round)
        cluster.print_freq_graph(round, False)

        while 1:
            print ("\nGreedy round %s:------------------------------" \
                    % u.id_round[round])
            greedy = cluster.greedy_new_cluster(round)
            if greedy and round < 26:
                cluster.cluster_lst = cluster.new_cluster_lst[:]
                cluster.print_cluster(round, False)

                cluster.print_greedy_backtrace(round, False)
                cluster.greedy_backtrace(round, False)
                build_complete_backtrace(round, cluster)

                # UNFOLDING
                if unfolding:
                    unfold = cluster.unfold_single_clusters(round)
                    if unfold:
                        cluster.cluster_lst = cluster.new_cluster_lst[:]

                round += 1
                cluster.build_freq_graph(round)
                cluster.print_freq_graph(round, False)

            else:
                # temp
                cluster.final_assembly(round - 1, etree, dbgr)
                cluster.chunking_clusters(etree, dbgr)

                if build_sections:
                    cluster.chunking_sections(etree, dbgr, section)

                print("\nClustering done\n")
                sys.exit(2)