def get_phf(ruleset): """ Generate number of states, transitions and consumed memory for \ Perfect hashing DFA. """ # Create parser - use default parser po = parser.parser() # Parse input file po.load_file(ruleset) # create phf_dfa automaton aut = PHF_DFA() # Make automaton from RE which was in input file aut.create_by_parser(po) # redefine default PHF class a = bdz() a.set_ratio(2.0) aut.set_PHF_class(a) # compute dfa and PHF table aut.compute() # Return experimental results return [ "Perfect Hashing DFA", aut.get_state_num(), aut.get_trans_num(), aut.report_memory_real(), aut.report_memory_real() ]
def get_hfa(ruleset): """ Generate number of states, transitions and consumed memory for \ History FA. """ # Create parser - use default parser po = parser.parser() # Parse input file po.load_file(ruleset) # Create History FA object history = HistoryFA() # Make automaton from RE which was in input file history.create_by_parser(po) # Remove epsilons history.remove_epsilons() # Store the NFA NFA = history.get_automaton(True) # Determinise the automaton history.determinise(create_table=True) # Create History FA history.compute(NFA) # Return experimental results return [ "History FA", history.get_state_num(), history.get_trans_num(), history.report_memory_optimal(), history.report_memory_naive() ]
def get_hybfa(ruleset): """ Generate number of states, transitions and consumed memory for \ History FA. """ # Create parser - use default parser po = parser.parser() # Parse input file po.load_file(ruleset) # Create Hybrid FA object hyb_fa = hybrid_fa() # Make automaton from RE which was in input file hyb_fa.create_by_parser(po) # set parameters for _is_special() function hyb_fa.set_max_head_size(-1) # off hyb_fa.set_max_tx(-1) # off hyb_fa.set_special_min_depth(2) # Create Hybrid FA hyb_fa.compute() # Return experimental results return [ "Hybrid FA", hyb_fa.get_state_num(), hyb_fa.get_trans_num(), hyb_fa.report_memory_optimal(), hyb_fa.report_memory_naive() ]
def get_ddfa(ruleset): """ Generate number of states, transitions and consumed memory for DDFA. """ # Create parser - use default parser po = parser.parser() # Parse input file po.load_file(ruleset) # Create Delay DFA object DelayDfa = DELAY_DFA() # Make automaton from RE which was in input file DelayDfa.create_by_parser(po) # Make Delay DFA # Resolve alphabet DelayDfa.resolve_alphabet() # Create Delay DFA DelayDfa.compute() # Return experimental results return [ "Delay DFA", DelayDfa.get_state_num(), DelayDfa.get_trans_num(), DelayDfa.report_memory_optimal(), DelayDfa.report_memory_naive() ]
def get_hcfa(ruleset): """ Generate number of states, transitions and consumed memory for History\ counting FA. """ # Create parser - use default parser po = parser.parser("pcre_parser", True) # Parse input file po.load_file(ruleset) # Create History Counting FA object history_counting = HistoryCountingFA() # Make automaton from RE which was in input file history_counting.create_by_parser(po) # Remove epsilons history_counting.remove_epsilons() # Store the NFA NFA = history_counting.get_automaton(True) # Replace X{m,n} with X* NFA_without_cnt = \ history_counting._replace_length_restriction_with_a_closure(NFA) NFA = history_counting.get_automaton(True) # Assingn the automaton with replaced X{m,n} with X* history_counting._automaton = NFA_without_cnt # Determinise the automaton history_counting.determinise(create_table = True) # Create History Counting FA history_counting.compute(NFA) # Return experimental results return ["History Counting FA", history_counting.get_state_num(), history_counting.get_trans_num(), history_counting.report_memory_optimal(), history_counting.report_memory_naive()]
def generate_text(rule): """ From given regular expression (rule) generate corresponding string. """ aut = b_nfa() par = parser("pcre_parser") par.set_text(rule) if not aut.create_by_parser(par): return "" aut.remove_epsilons() aut.search("a") state = 0 string = "" while not state in aut._automaton.final: trans = aut._mapper[state] rnd1 = random.randint(0, len(trans) - 1) sym = aut._automaton.alphabet[list(trans)[rnd1][0]] if sym.get_type() == b_symbol.io_mapper["b_Sym_char_class"]: chars = sym.charClass else: chars = sym.char state = list(trans)[rnd1][1] rnd2 = random.randint(0, len(chars) - 1) string += list(chars)[rnd2] if 1 not in aut.search(string): print "FAIL" return string
def get_hcfa(ruleset): """ Generate number of states, transitions and consumed memory for History\ counting FA. """ # Create parser - use default parser po = parser.parser("pcre_parser", True) # Parse input file po.load_file(ruleset) # Create History Counting FA object history_counting = HistoryCountingFA() # Make automaton from RE which was in input file history_counting.create_by_parser(po) # Remove epsilons history_counting.remove_epsilons() # Store the NFA NFA = history_counting.get_automaton(True) # Replace X{m,n} with X* NFA_without_cnt = \ history_counting._replace_length_restriction_with_a_closure(NFA) NFA = history_counting.get_automaton(True) # Assingn the automaton with replaced X{m,n} with X* history_counting._automaton = NFA_without_cnt # Determinise the automaton history_counting.determinise(create_table=True) # Create History Counting FA history_counting.compute(NFA) # Return experimental results return [ "History Counting FA", history_counting.get_state_num(), history_counting.get_trans_num(), history_counting.report_memory_optimal(), history_counting.report_memory_naive() ]
def main(): try: opts, args = getopt.getopt( sys.argv[1:], "ho:d:si:S:", ["help", "output=", "display=", "simple", "input=", "style="]) except getopt.GetoptError as err: # print help information and exit: print str(err) usage() sys.exit(1) output_file = None input_file = None display_file = None simple = False fsm_style = "nfa" for o, a in opts: if o == "-s": simple = True elif o in ("-h", "--help"): usage() sys.exit() elif o in ("-o", "--output"): output_file = a elif o in ("-i", "--input"): input_file = a elif o in ("-d", "--display"): display_file = a elif o in ("-S", "--style"): fsm_style = a.lower() else: assert False, "unhandled option " + o if (input_file == None or output_file == None): print "Input file with PCRE patterns and output file for FSM must be specified!" sys.exit(1) fsm = b_automaton.b_Automaton() p = parser.parser("pcre_parser") p.load_file(input_file) fsm.create_by_parser(p) if fsm_style == "nfa": fsm.remove_epsilons() elif fsm_style == "pa": fsm.thompson2glushkov() elif fsm_style == "dpa": fsm.thompson2reverse_glushkov() else: print "Unsupported automaton type: " + fsm_style print "Supported automata styles are: nfa, pa, dpa" print "See " + sys.argv[0] + " -h for details" sys.exit(1) if simple == True: fsm.remove_char_classes() fsm.save_to_timbuk(output_file, simple) if display_file != None: fsm.show(display_file, " ")
def main(): try: opts, args = getopt.getopt(sys.argv[1:], "ho:d:si:S:", ["help", "output=", "display=", "simple", "input=", "style="]) except getopt.GetoptError as err: # print help information and exit: print str(err) usage() sys.exit(1) output_file = None input_file = None display_file = None simple = False fsm_style = "nfa" for o, a in opts: if o == "-s": simple = True elif o in ("-h", "--help"): usage() sys.exit() elif o in ("-o", "--output"): output_file = a elif o in ("-i", "--input"): input_file = a elif o in ("-d", "--display"): display_file = a elif o in ("-S", "--style"): fsm_style = a.lower() else: assert False, "unhandled option " + o if (input_file == None or output_file == None): print "Input file with PCRE patterns and output file for FSM must be specified!" sys.exit(1) fsm = b_automaton.b_Automaton() p = parser.parser("pcre_parser") p.load_file(input_file) fsm.create_by_parser(p) if fsm_style == "nfa": fsm.remove_epsilons() elif fsm_style == "pa": fsm.thompson2glushkov() elif fsm_style == "dpa": fsm.thompson2reverse_glushkov() else: print "Unsupported automaton type: " + fsm_style print "Supported automata styles are: nfa, pa, dpa" print "See " + sys.argv[0] + " -h for details" sys.exit(1) if simple == True: fsm.remove_char_classes() fsm.save_to_timbuk(output_file, simple) if display_file != None: fsm.show(display_file, " ")
def test_report_memory_naive(self): """report_memory_naive()""" # /ab[^1234]*cd|efg/; test with an expression containing one # alternation [^1234]*, the second is not par = parser("pcre_parser") par.set_text("/ab[^1234]*cd|efg/") history = HistoryFA() history.create_by_parser(par) history.remove_epsilons() NFA = history.get_automaton(True) history.determinise(create_table = True) history.compute(NFA) self.assertTrue(history.report_memory_naive() == 187)
def get_sb(ruleset): # Create sourdis_bispo_nfa object cn = sourdis_bispo_nfa.sourdis_bispo_nfa() # Preprocess the REs preprocessed = cn.find_pcre_repetitions(ruleset) # Create parser - use default parser Test0 = parser.parser("pcre_parser") # Load REs Test0.set_text(preprocessed) # Parse RE and create NFA cn.create_by_parser(Test0) # Call the compute method cn.compute() # Get number of used LUTs and FFs data = cn.report_logic() return ["Sourdis Bispo", data[0], data[1]]
def main(): try: opts, args = getopt.getopt( sys.argv[1:], "ho:d:si:S:", ["help", "output=", "display=", "simple", "input=", "style="]) except getopt.GetoptError as err: # print help information and exit: print str(err) usage() sys.exit(1) output_file = None input_file = None display_file = None simple = False fsm_style = "nfa" for o, a in opts: if o == "-s": simple = True elif o in ("-h", "--help"): usage() sys.exit() elif o in ("-o", "--output"): output_file = a elif o in ("-i", "--input"): input_file = a elif o in ("-d", "--display"): display_file = a else: assert False, "unhandled option " + o if (input_file == None or output_file == None): print "Input file with PCRE patterns and output file for FSM must be specified!" sys.exit(1) fsm = b_automaton.b_Automaton() p = parser.parser("pcre_parser") p.load_file(input_file) fsm.create_by_parser(p) fsm.remove_epsilons() print "State num: ", fsm.get_state_num() if simple == True: fsm.remove_char_classes() fsm.save_to_FA_format(output_file) if display_file != None: fsm.show(display_file, " ")
def get_sp_s(ruleset, stride): """ Generate number of LUTs and FFs for strided Sidhu and Prasana's approach. """ # Create sindhu_prasana_nfa object cn = sindhu_prasana_nfa.sindhu_prasana_nfa(stride) # Create parser - use default parser Test0 = parser.parser() # Load RE file Test0.load_file(ruleset) # Parse RE and create NFA cn.create_by_parser(Test0) # Call the compute method cn.compute() # Get number of used LUTs and FFs data = cn.report_logic() return ["Sidhu Prasana with stride ", stride, data[0], data[1]]
def get_hybfas(ruleset): """ Generate number of states, transitions and consumed memory for \ History FA (Suchodol). """ # Create parser - use default parser po = parser.parser() # Create Hybrid FA object hyb_fa = JHybridFA() # Set parser hyb_fa.set_parser(po) # Parse input file hyb_fa.load_file(ruleset) # Create Hybrid FA hyb_fa.compute() # Return experimental results return ["Hybrid FA (Suchodol)", hyb_fa.get_state_num(), hyb_fa.get_trans_num(), hyb_fa.report_memory_optimal(), hyb_fa.report_memory_naive()]
def main(): try: opts, args = getopt.getopt(sys.argv[1:], "ho:d:si:S:", ["help", "output=", "display=", "simple", "input=", "style="]) except getopt.GetoptError as err: # print help information and exit: print str(err) usage() sys.exit(1) output_file = None input_file = None display_file = None simple = False fsm_style = "nfa" for o, a in opts: if o == "-s": simple = True elif o in ("-h", "--help"): usage() sys.exit() elif o in ("-o", "--output"): output_file = a elif o in ("-i", "--input"): input_file = a elif o in ("-d", "--display"): display_file = a else: assert False, "unhandled option " + o if (input_file == None or output_file == None): print "Input file with PCRE patterns and output file for FSM must be specified!" sys.exit(1) fsm = b_automaton.b_Automaton() p = parser.parser("pcre_parser") p.load_file(input_file) fsm.create_by_parser(p) fsm.remove_epsilons() print "State num: ", fsm.get_state_num() if simple == True: fsm.remove_char_classes() fsm.save_to_FA_format(output_file) if display_file != None: fsm.show(display_file, " ")
def get_clark_s(ruleset, stride): """ Generate number of LUTs and FFs for strided Clark's approach with char classes. """ # Create clark_nfa object cn = clark_nfa.clark_nfa(False, stride) # Create parser - use default parser Test0 = parser.parser("pcre_parser") # Load RE file Test0.load_file(ruleset) # Parse RE and create NFA cn.create_by_parser(Test0) # Call the compute method cn.compute() # Get number of used LUTs and FFs data = cn.report_logic() return ["Clark with stride ", stride, data[0], data[1]]
def get_ddfa(ruleset): """ Generate number of states, transitions and consumed memory for DDFA. """ # Create parser - use default parser po = parser.parser() # Parse input file po.load_file(ruleset) # Create Delay DFA object DelayDfa = DELAY_DFA() # Make automaton from RE which was in input file DelayDfa.create_by_parser(po) # Make Delay DFA # Resolve alphabet DelayDfa.resolve_alphabet() # Create Delay DFA DelayDfa.compute() # Return experimental results return ["Delay DFA", DelayDfa.get_state_num(), DelayDfa.get_trans_num(), DelayDfa.report_memory_optimal(), DelayDfa.report_memory_naive()]
def test_strided(self): """ Tests the algorithm with stride = 2. """ # Create sindhu_prasana_nfa object cn = sindhu_prasana_nfa.sindhu_prasana_nfa(2) # Create parser - use default parser Test0 = parser.parser("pcre_parser") # Load RE file Test0.load_file(aux_func.getPatternMatchDir() + "/algorithms/sindhu_prasana_nfa/test/tests/test.pcre") # Parse RE and create NFA cn.create_by_parser(Test0) # Call the compute method cn.compute() # Create the simulation create_simulation(cn) # run the simulation run_simulation(self)
def test_strided(self): """ Tests the algorithm with stride = 2. """ # Get test directory tdir = aux_func.getPatternMatchDir() + "/algorithms/clark_nfa/test/" # Create clark_nfa object cn = clark_nfa.clark_nfa(False, 2) # Create parser - use default parser Test0 = parser.parser("pcre_parser") # Load RE file Test0.load_file(tdir + "tests/test.pcre") # Parse RE and create NFA cn.create_by_parser(Test0) # Call the compute method cn.compute() # Create the simulation create_simulation(cn) # run the simulation run_simulation(self)
def get_phf(ruleset): """ Generate number of states, transitions and consumed memory for \ Perfect hashing DFA. """ # Create parser - use default parser po = parser.parser() # Parse input file po.load_file(ruleset) # create phf_dfa automaton aut = PHF_DFA() # Make automaton from RE which was in input file aut.create_by_parser(po) # redefine default PHF class a = bdz() a.set_ratio(2.0) aut.set_PHF_class(a) # compute dfa and PHF table aut.compute() # Return experimental results return ["Perfect Hashing DFA", aut.get_state_num(), aut.get_trans_num(), aut.report_memory_real(), aut.report_memory_real()]
def get_hybfa(ruleset): """ Generate number of states, transitions and consumed memory for \ History FA. """ # Create parser - use default parser po = parser.parser() # Parse input file po.load_file(ruleset) # Create Hybrid FA object hyb_fa = hybrid_fa() # Make automaton from RE which was in input file hyb_fa.create_by_parser(po) # set parameters for _is_special() function hyb_fa.set_max_head_size(-1) # off hyb_fa.set_max_tx(-1) # off hyb_fa.set_special_min_depth(2) # Create Hybrid FA hyb_fa.compute() # Return experimental results return ["Hybrid FA", hyb_fa.get_state_num(), hyb_fa.get_trans_num(), hyb_fa.report_memory_optimal(), hyb_fa.report_memory_naive()]
def test_sb(self): """ Tests the Sordis-Bispo algorithm. """ # Create sourdis_bispo_nfa object cn = sourdis_bispo_nfa.sourdis_bispo_nfa() # Preprocess the REs preprocessed = cn.find_pcre_repetitions( aux_func.getPatternMatchDir() + "/algorithms/sourdis_bispo_nfa/test/tests/test.pcre") # Create parser - use default parser Test0 = parser.parser("pcre_parser") # Load REs Test0.set_text(preprocessed) # Parse RE and create NFA cn.create_by_parser(Test0) # Call the compute method cn.compute() # Create the simulation create_simulation(cn) # run the simulation run_simulation(self)
def get_hfa(ruleset): """ Generate number of states, transitions and consumed memory for \ History FA. """ # Create parser - use default parser po = parser.parser() # Parse input file po.load_file(ruleset) # Create History FA object history = HistoryFA() # Make automaton from RE which was in input file history.create_by_parser(po) # Remove epsilons history.remove_epsilons() # Store the NFA NFA = history.get_automaton(True) # Determinise the automaton history.determinise(create_table = True) # Create History FA history.compute(NFA) # Return experimental results return ["History FA", history.get_state_num(), history.get_trans_num(), history.report_memory_optimal(), history.report_memory_naive()]
def get_hybfas(ruleset): """ Generate number of states, transitions and consumed memory for \ History FA (Suchodol). """ # Create parser - use default parser po = parser.parser() # Create Hybrid FA object hyb_fa = JHybridFA() # Set parser hyb_fa.set_parser(po) # Parse input file hyb_fa.load_file(ruleset) # Create Hybrid FA hyb_fa.compute() # Return experimental results return [ "Hybrid FA (Suchodol)", hyb_fa.get_state_num(), hyb_fa.get_trans_num(), hyb_fa.report_memory_optimal(), hyb_fa.report_memory_naive() ]
""" if __name__ == '__main__': # Clark mapping method without suppotr for char classes print("-------------------------------------------------------------------") print(" Example of use: Clark NFA ") print("-------------------------------------------------------------------") print(" Ruleset: ../../rules/L7/selected.pcre ") print(" Character Classes: No ") print(" Strided: No ") print(" Genereated VHDL output: clark_nfa_impl_wochc.vhd ") print("-------------------------------------------------------------------") # Create clark_nfa object cn = clark_nfa.clark_nfa(True) # Create parser - use default parser Test0 = parser.parser("pcre_parser") # Load RE file Test0.load_file("../../rules/L7/selected.pcre") # Parse RE and create NFA cn.create_by_parser(Test0) # Call the compute method cn.compute() # Get number of used LUTs and FFs data = cn.report_logic() print(" Used LUTs estimation: " + str(data[0])) print(" Used FFs estimation: " + str(data[1])) print("-------------------------------------------------------------------") # Save implementation # Open file f = open("clark_nfa_impl_wochc.vhd", "w") # Get VHDL code and write the code
def test(): """ Run searching using pcregrep and PHF_DFA. Prints out the results. """ # parse options usage = "usage: %prog rules.pcre pcap_dir/ [options]" optparser = OptionParser(usage=usage) optparser.add_option("-O", "--outputfile", dest="resultfile", help="output file for results, default is stdout") optparser.add_option("-P", "--PerPacket", dest="PerPacket", action="store_true", default=False, help="compare nonfaulty matching for flows and packets, faulty algorithm is used only with flows") optparser.add_option("-s", "--showprogress", dest="progress", action="store_true", default=False, help="show progress of computation") optparser.add_option("-C", "--count", dest="maxiter", type="int", default="1", help="number of test iterations") optparser.add_option("-F", "--faulty", dest="FAULTY", type="int", default="0", help="number of bits for compress hash, default is 0 (no faulty transitions)") optparser.add_option("-D", "--debuglevel", dest="DEBUG", type="int", default="0", help="debug output level (0-2)") optparser.add_option("-S", "--savefile", dest="savefile", default="", metavar="FILE", help="save nfa_data in FILE") optparser.add_option("-L", "--loadfile", dest="autfile", default="", metavar="FILE", help="load nfa_data from FILE") optparser.add_option("-N", "--nonfaulty", dest="NonFaulty", action="store_true", default=False, help="try to generate PHF table without collisions, therefore ensure nonfaulty matching. Experimental code. " "May take a long time with small compress hash output.") (options, args) = optparser.parse_args() global FAULTY, DEBUG if len(args) != 2: print "You must specify rules.pcre and pcap_dir/" optparser.print_usage() exit(1) rulesfile, inputdir = args PerPacket, resultfile, maxiter, autfile, savefile, FAULTY, DEBUG = options.PerPacket, options.resultfile, options.maxiter, options.autfile, options.savefile, options.FAULTY, options.DEBUG progress = options.progress NonFaulty = options.NonFaulty if inputdir[-1] == "/": inputdir = inputdir[:-1] # remove '/' from the end rules = open(rulesfile, 'rb') if PerPacket: packetdir = inputdir + "/packets" inputdir = inputdir + "/flows" if resultfile: sys.stdout = open(resultfile, 'a') totalhits, totalfp, totalfn = (0, 0, 0) iter = 0 while iter != maxiter: if progress: print >>sys.stderr, "\r", ' '*80, '\r',"pcregrep", if not iter: # prepare pcregrep p = subprocess.Popen("cd pcre-8.20/ && make pcregrep", shell=True, stdout=subprocess.PIPE) p.wait() results = dict() file_list = list() rule_count = len(open(rulesfile).readlines()) for root, dirs, files in os.walk(inputdir): for i in files: i = os.path.join(root, i) file_list.append(i) if not iter: results[i] = [rule_count*[0],rule_count*[0],rule_count*[0]] else: results[i][0] = rule_count*[0] #results = init_results rule_num = 0 grep_reg_exp = "grep_reg_exp." + str(os.getpid()) for rule in rules: if not iter: if DEBUG: print rule, (grep_rule, grep_params) = parse_rule(rule) f = open(grep_reg_exp, 'w') f.write(grep_rule) f.close() p = subprocess.Popen("pcre-8.20/pcregrep --buffer-size 50000 --color=auto -N ANYCRLF" + grep_params + " -r -l -f " + grep_reg_exp + " " + inputdir, shell=True, stdout=subprocess.PIPE) p.wait() for out in p.stdout: item = out.split()[0] results[item][1][rule_num] = 1 if PerPacket: p = subprocess.Popen("pcre-8.20/pcregrep --buffer-size 50000 --color=auto -N ANYCRLF" + grep_params + " -r -l -f " + grep_reg_exp + " " + packetdir, shell=True, stdout=subprocess.PIPE) p.wait() for out in p.stdout: item = inputdir + "/" + out.split()[0].split("-")[1].replace("_", "/") results[item][2][rule_num] = 1 rule_num += 1 try: os.remove(grep_reg_exp) except: pass if progress: print >>sys.stderr, "\r", ' '*80, '\r', "create automaton", #aut = b_Automaton() aut = PHF_DFA() if autfile: aut.create_from_nfa_data(nfa_data().load_from_file(autfile)) else: par = parser("pcre_parser") #par.set_text(rule) par.load_file(rulesfile) aut.create_by_parser(par) if DEBUG: aut.show("NFA.dot") #aut.remove_epsilons() if progress: print >>sys.stderr, "\r", ' '*80, '\r', "resolve alphabet", aut.resolve_alphabet() if progress: print >>sys.stderr, "\r", ' '*80, '\r', "determinise", aut.determinise() if progress: print >>sys.stderr, "\r", ' '*80, '\r', "minimise", aut.minimise() if DEBUG: aut.show("DFA.dot") if savefile: aut._automaton.save_to_file(savefile) aut._automaton1 = aut._automaton aut.set_table_parameters((20,10)) if DEBUG > 1: print "Without fallback state:" print "Symbols:", len(aut._automaton.alphabet) print "States:", len(aut._automaton.states) print "Transitions:", aut.get_trans_num(), float(aut.get_trans_num()) / (aut.get_state_num() * aut.get_alpha_num()) * 100, "%" if isinstance(aut, PHF_DFA): if progress: print >>sys.stderr, "\r", ' '*80, '\r', "generate PHF", if aut.get_trans_num() == (aut.get_state_num() * aut.get_alpha_num()): aut.enable_fallback_state(warning=False) if FAULTY: aut.enable_faulty_transitions(FAULTY) if NonFaulty: aut.enable_faulty_check() aut.compute() if DEBUG: print "Fallback state:", aut.fallback_state print "Symbols:", len(aut._automaton.alphabet) print "States:", len(aut._automaton.states) print "Transitions:", aut.get_trans_num(), float(aut.get_trans_num()) / (aut.get_state_num() * aut.get_alpha_num()) * 100, "%" count = 1 all = len(file_list) if progress: print >> sys.stderr, '\r' + 80*' ' + '\r', for f in file_list: # progress if progress: print >> sys.stderr, '\r', print >> sys.stderr, str(iter+1)+'/'+str(maxiter)+ ":", count, '/', all, # sys.stderr.flush() count += 1 data = open(f, 'rb').read() results[f][0] = aut.search(data) if progress: print >>sys.stderr, "\r", ' '*80, '\r', "compare results", if isinstance(aut, PHF_DFA) and DEBUG: if DEBUG > 1: print "List of collisions:" print aut.collisions for tran, i in aut.collisions.iteritems(): #print tran, i print BitArray(bytes=tran[0], length=aut.symbol_bits).uint, BitArray(bytes=tran[1], length=aut.state_bits).uint, i print "SYM:", aut._automaton.alphabet[BitArray(bytes=tran[0], length=aut.symbol_bits).uint] print "Bad transitions:", aut.bad_transitions print "Collisions:", len(aut.collisions) print "Compress bits:", aut.compress_bits stats = compare_results(results) stats = list(stats) if stats[0] == 0: print "Zero hits, cannot compute F-measure!" stats[0] = 1 if DEBUG: print "Total number of searched packets/flows:", stats[3] print "Hits:", stats[0] totalhits += stats[0] totalfp += stats[1] totalfn += stats[2] precis = float(stats[0])/(stats[1]+stats[0]) recall = float(stats[0])/(stats[0]+stats[2]) fmeas = 2* precis * recall / (precis + recall) print "False positives:", stats[1], precis*100, "%" print "False negatives:", stats[2], recall*100, "%" print "F-measure:", fmeas*100, "%" if PerPacket: print "Per packet errors:", stats[4], stats[5] print '-'*80 iter += 1 print "Total stats:" precis = float(totalhits)/(totalfp + totalhits) recall = float(totalhits)/(totalfn + totalhits) fmeas = 2* precis * recall / (precis + recall) print "Hits:", totalhits print "False positives:", totalfp, precis*100, "%" print "False negatives:", totalfn, recall*100, "%" print "F-measure:", fmeas*100, "%" print "_"*80
def _make_his_fa(self, file_name): """ Fuction for make History based FA from RE in FileName. :param file_name: Name of input file :type file_name: string """ # Discover before part, according .* or .{m} like pattern, # and change .{m} to .* like. b = [] # keys for closure states cl = [] # closure NFA states cn = [] # counters for counting constraint cn_i = [] # counters index fr = open(file_name, "r") tmp = tempfile.NamedTemporaryFile(delete=False) for line in fr.readlines(): # remove '\n' from end of line line = line.rsplit('\n', 1)[0] pattern = re.compile( r""" (?<=[^/])[.](?=[*]) # for like .* (?![^[]*?\]) # because of .* not in [.*] | \[\^ .*? # for like [^abc]* (?<!\\)\](?=[*]) (?![^[]*?\]) | [.](?=[{]) # for like .{15} (?![^[]*?\]) | .(?=[{]) # for like a{15} (?![^[]*?\]) | \[\^? # for like [abc]{15} or [^abc]{15} .*? (?<!\\)\](?=[{]) (?![^[]*?\]) """, re.X) # split line to before (split[0]) and after (split[1]) part split = re.split(pattern, line, maxsplit=1) # remove .* from begin of pattern if split[0].find(".*") != -1: split[0] = '/' + split[0][split[0].find(".*") + 2:] # line contain .* or .{m} like pattern if len(split) == 2: b.append(split[0][1:]) cl.append([]) if split[1][0] == '{': cn.append(int(split[1][1:split[1].find('}', 1)])) cn_i.append(len(cn) - 1) # replace .{m} like to .* like line = line[:line.find(split[1])] + '*' + \ split[1][split[1].find('}') + 1:] else: cn.append(-1) # append line to tmp file tmp.write(line + '\n') fr.close() tmp.close() # Make DFA. # Parse input file par = parser("pcre_parser") par.load_file(tmp.name) # Make automat from RE which was in input file self.create_by_parser(par) # Make Deterministic FA self.determinise(True) # remove temporary file os.unlink(tmp.name) # Adjustment _state_representation list. m = list(min(self._state_representation[1:])) r = self._state_representation for i in range(0, len(r)): for x in m: if x in r[i]: r[i].remove(x) r[i] = list(r[i]) # Discover closure state. a = self._automaton cur_state = a.start # sort transitions sort = {} # sorted transitions for s in range(0, len(a.states)): sort[s] = [] for t in a.transitions: sort[t[0]].append(t[1:]) for i in range(0, len(b)): for c in b[i]: for t in sort[cur_state]: prev_state = cur_state # single character if isinstance(a.alphabet[t[0]], sym_char.b_Sym_char): if a.alphabet[t[0]].char == c: cur_state = t[-1] # skip other transitions break # character range else: if c in a.alphabet[t[0]].charClass: cur_state = t[-1] # skip other transitions break # remove closure transition a.transitions.remove((prev_state, t[0], t[1])) cl[i] = r[cur_state][-1] + 1 # append closure history transition if cn[i] == -1: a.transitions.add((prev_state, t[0], t[1], '|', -2, i)) else: # counting constraint a.transitions.add( (prev_state, t[0], t[1], '|', -2, i, cn[i], cn_i.index(i))) cur_state = a.start # Discover fading states and their overlap states. f = [] # fading states (DFA) f_d = {} # fading states in dictionary, key is string before o = [] # overlap states according fading states (DFA) for i in range(0, len(b)): f_d[b[i]] = [] c_s = cl[i] # c_s is closure states (NFA) for j in range(0, len(r)): if c_s in r[j]: over = list(r[j]) for s in cl: if s in over: over.remove(s) # append only states which have overlap state if over in r: if j not in f: f.append(j) f_d[b[i]].append(j) # append overlap DFA state o.append(r.index(over)) # Remove fading states. for s in f: del a.states[s] # delete deleted states from finite states for s in list(a.final): if s in f: a.final.remove(s) # change numbering for states a_s = sorted(a.states.keys()) # aux. states tmp = {} for k in a_s: tmp[a_s.index(k)] = a.states[k] a.states = tmp # change numbering for final states tmp = set() for s in a.final: tmp.add(a_s.index(s)) a.final = tmp # change numbering for start state a.start = a_s.index(a.start) # Change fading transitions. # transition is tuple of 3 numbers plus history properties # like: (S, A, D) # S is source state # A is alphabet (transition char) # D is destination state # like: (S, A, D, -2, f_i, '|', -4, f_i) # -2 is flag which is needed for execution transition # f_i is flag index # '|' separating needed properties for execution tran. of properties # which will be set after tran. finished # -4 flag which will be set after tran. finish # f_i is flag index # flags: -2 = set, -3 = must be set, -4 = reset, -5 = must be reset #c_s_D = [] # closure state DFA ! #print "cl:",cl #print "r:",r #for c_s_N in cl: #print "c_s_N:",c_s_N #c_s_D.append(r.index([c_s_N])) aux = set() d = {} # transitions for pruning process tmp = [] # helpful list for pruning process tmp_t = [] # helpful list for pruning process for t in a.transitions: t = list(t) # destination state is fading state if t[2] in f: # source state is fading state if t[0] in f: # this transition only if is set flag (=) # append must be set flag for i in range(0, len(b)): if t[0] in f_d[b[i]]: t.append(-3) t.append(i) if cn[i] != -1: # append counting constraint counter 0 t.append(0) t.append(cn_i.index(i)) t.append('|') # change to overlap state t[0] = o[f.index(t[0])] # source state is non fading else: if '|' not in t: t.append('|') # append set flag for i in range(0, len(b)): if t[2] in f_d[b[i]]: t.append(-2) t.append(i) if cn[i] != -1: # append counting constraint set counter t.append(cn[i]) t.append(cn_i.index(i)) # change des. state to overlap state t[2] = o[f.index(t[2])] # source state is fading state elif t[0] in f: # append must be set flag for i in range(0, len(b)): if t[0] in f_d[b[i]]: t.append(-3) t.append(i) if cn[i] != -1: # append counting constraint counter 0 t.append(0) t.append(cn_i.index(i)) t.append('|') # destination state is non-fading state # reset flag for i in range(0, len(b)): if t[0] in f_d[b[i]]: t.append(-4) t.append(i) if cn[i] != -1: # append counting constraint reset counter t.append(0) t.append(cn_i.index(i)) # change to overlap state t[0] = o[f.index(t[0])] # change numbering t[0] = a_s.index(t[0]) t[2] = a_s.index(t[2]) aux.add(tuple(t)) # for pruning process if len(t) == 3: t_a = t + ['|'] else: t_a = t[0:3] + t[t.index('|'):] t_a = tuple(t_a) if t_a in tmp: # select similarly transitions if not d.has_key(t_a): d[t_a] = [tmp_t[tmp.index(t_a)]] if t not in d[t_a]: d[t_a].append(t) else: tmp.append(t_a) tmp_t.append(t) a.transitions = aux # Pruning process (remove similarly transitions). for key in d.keys(): # exist transition without condition if list(key[0:3]) in d[key]: # keep only this transition d[key].remove(list(key[0:3])) for t in d[key]: a.transitions.remove(tuple(t)) else: # find set and remove all her subset c = [] # auxiliary, conditions for tran. c_l = [] # auxiliary, conditions length tran = [] # auxiliary, original tran. for t in d[key]: aux = [] for i in range(3, t.index('|'), 2): aux += [tuple(t[i:i + 2])] c.append(aux) c_l.append(len(aux)) tran.append(tuple(t)) while len(c) != 0: s = c[c_l.index(min(c_l))] # set for t_c in list(c): flag = 1 for x in s: if x not in t_c: flag = 0 break # tran. fall into this set if flag and s != t_c: # remove subset tran. del c_l[c.index(t_c)] a.transitions.remove(tran[c.index(t_c)]) c.remove(t_c) # remove set (only from c) del c_l[c.index(s)] c.remove(s) # Append must be reset flag. c = {} # tran. which have must be set flag c_i = {} # indexes of flags n = {} # tran. which have not must be set flag for s in range(0, len(a.states)): c[s] = [] c_i[s] = [] n[s] = [] for t in a.transitions: added = False t = list(t) # must be set tran. if len(t) > 3 and (t.index('|') - 3) > 0: tmp = [] for i in range(3, t.index('|'), 2): if t[i] == -3: tmp.append(t[i + 1]) if tmp != []: if tuple(t) not in c[t[0]]: c[t[0]].append(tuple(t)) c_i[t[0]].append(tmp) added = True # not must be set tran. if not added: if tuple(t) not in n[t[0]]: n[t[0]].append(tuple(t)) for s in range(0, len(a.states)): for t in c[s]: # discover tran. char if isinstance(a.alphabet[t[1]], sym_char.b_Sym_char): chars = list(a.alphabet[t[1]].char) else: chars = list(a.alphabet[t[1]].charClass) for t_n in list(n[s]): if isinstance(a.alphabet[t_n[1]], sym_char.b_Sym_char): chars_n = list(a.alphabet[t_n[1]].char) else: chars_n = list(a.alphabet[t_n[1]].charClass) for char in chars_n: if char in chars: # remove bad tran. a.transitions.remove(t_n) n[s].remove(t_n) # append must be reset flag t_n = list(t_n) if '|' not in t_n: for i in c_i[s][c[s].index(t)]: t_n.append('-5') t_n.append(i) t_n.append('|') else: for i in c_i[s][c[s].index(t)]: t_n.insert(3, '-5') t_n.insert(4, i) a.transitions.add(tuple(t_n)) break # Count of flags and counters. self.flag_c = len(b) self.ctr_c = len(cn_i) self.cn_i = cn_i # Add automat flag. a.Flags["History FA"] = True
def test_search(self): """search()""" # For RE upstairs do tests for this method. # /abcd/; test with an expression that does not use properties # of HistoryFA par = parser("pcre_parser") par.set_text("/abcd/") history = HistoryFA() history.create_by_parser(par) history.remove_epsilons() NFA = history.get_automaton(True) history.determinise(create_table = True) history.compute(NFA) self.assertTrue(history.search("efg") == [0]) self.assertTrue(history.search("efabcg") == [0]) self.assertTrue(history.search("efabcd") == [1]) self.assertTrue(history.search("abcefabcdabcx") == [1]) self.assertTrue(history.search("abcd") == [1]) self.assertTrue(history.search("123abcd456") == [1]) self.assertTrue(history.search("123ab0cd456") == [0]) # /ab.*cd /; test with an expression that contains .* par = parser("pcre_parser") par.set_text("/ab.*cd/") history = HistoryFA() history.create_by_parser(par) history.remove_epsilons() NFA = history.get_automaton(True) history.determinise(create_table = True) history.compute(NFA) self.assertTrue(history.search("efg") == [0]) self.assertTrue(history.search("abnecoefg") == [0]) self.assertTrue(history.search("abnecocx") == [0]) self.assertTrue(history.search("necoabnecocdneco") == [1]) self.assertTrue(history.search("abcd") == [1]) self.assertTrue(history.search("123abcd456") == [1]) self.assertTrue(history.search("123ab0cd456") == [1]) # /ab[^1234]*cd|efg/; test with an expression containing one # alternation [^1234]*, the second is not par = parser("pcre_parser") par.set_text("/ab[^1234]*cd|efg/") history = HistoryFA() history.create_by_parser(par) history.remove_epsilons() NFA = history.get_automaton(True) history.determinise(create_table = True) history.compute(NFA) self.assertTrue(history.search("exfg") == [0]) self.assertTrue(history.search("abnecocx") == [0]) self.assertTrue(history.search("efg") == [1]) self.assertTrue(history.search("textefgtext") == [1]) self.assertTrue(history.search("abnecoefg") == [1]) self.assertTrue(history.search("necoabnecocdneco") == [1]) self.assertTrue(history.search("abcd") == [1]) self.assertTrue(history.search("123abcd456") == [1]) self.assertTrue(history.search("123ab0cd456") == [1]) self.assertTrue(history.search("ab1cd") == [0]) self.assertTrue(history.search("ab4cd") == [0]) self.assertTrue(history.search("abcefg") == [1]) self.assertTrue(history.search("123abblekekcd456") == [1]) self.assertTrue(history.search("123ab0cd456") == [1])
"-------------------------------------------------------------------") print( " Example of use: History FA ") print( "-------------------------------------------------------------------") print( " Ruleset: /ab.*cd/ ") print( " Graphical representation: automaton.dot ") print( " history.dot ") print( "-------------------------------------------------------------------") # Create PCRE parser object par = parser("pcre_parser") # Set the ruleset par.set_text("/ab.*cd/") # Create HistoryFA object history = HistoryFA() # Parse the ruleset history.create_by_parser(par) # Remove epsilons history.remove_epsilons() # Get copy of NFA part NFA = history.get_automaton(True) # Determinise the automaton. Record corespondance between DFA states # and NFA states history.determinise(create_table=True) # Create the History FA from deterministic automaton original NFA.
def _make_his_fa(self, file_name): """ Fuction for make History based FA from RE in FileName. :param file_name: Name of input file :type file_name: string """ # Discover before part, according .* or .{m} like pattern, # and change .{m} to .* like. b = [] # keys for closure states cl = [] # closure NFA states cn = [] # counters for counting constraint cn_i = [] # counters index fr = open(file_name, "r") tmp = tempfile.NamedTemporaryFile(delete=False) for line in fr.readlines(): # remove '\n' from end of line line = line.rsplit('\n', 1)[0] pattern = re.compile(r""" (?<=[^/])[.](?=[*]) # for like .* (?![^[]*?\]) # because of .* not in [.*] | \[\^ .*? # for like [^abc]* (?<!\\)\](?=[*]) (?![^[]*?\]) | [.](?=[{]) # for like .{15} (?![^[]*?\]) | .(?=[{]) # for like a{15} (?![^[]*?\]) | \[\^? # for like [abc]{15} or [^abc]{15} .*? (?<!\\)\](?=[{]) (?![^[]*?\]) """, re.X) # split line to before (split[0]) and after (split[1]) part split = re.split(pattern, line, maxsplit=1) # remove .* from begin of pattern if split[0].find(".*") != -1: split[0] = '/' + split[0][split[0].find(".*") + 2:] # line contain .* or .{m} like pattern if len(split) == 2: b.append(split[0][1:]) cl.append([]) if split[1][0] == '{': cn.append(int(split[1][1:split[1].find('}', 1)])) cn_i.append(len(cn) - 1) # replace .{m} like to .* like line = line[:line.find(split[1])] + '*' + \ split[1][split[1].find('}') + 1:] else : cn.append(-1) # append line to tmp file tmp.write(line + '\n') fr.close() tmp.close() # Make DFA. # Parse input file par = parser("pcre_parser") par.load_file(tmp.name) # Make automat from RE which was in input file self.create_by_parser(par) # Make Deterministic FA self.determinise(True) # remove temporary file os.unlink(tmp.name) # Adjustment _state_representation list. m = list(min(self._state_representation[1:])) r = self._state_representation for i in range(0, len(r)): for x in m: if x in r[i]: r[i].remove(x) r[i] = list(r[i]) # Discover closure state. a = self._automaton cur_state = a.start # sort transitions sort = {} # sorted transitions for s in range(0, len(a.states)): sort[s] = [] for t in a.transitions: sort[t[0]].append(t[1:]) for i in range(0, len(b)): for c in b[i]: for t in sort[cur_state]: prev_state = cur_state # single character if isinstance(a.alphabet[t[0]], sym_char.b_Sym_char): if a.alphabet[t[0]].char == c: cur_state = t[-1] # skip other transitions break # character range else : if c in a.alphabet[t[0]].charClass: cur_state = t[-1] # skip other transitions break # remove closure transition a.transitions.remove((prev_state, t[0], t[1])) cl[i] = r[cur_state][-1] + 1 # append closure history transition if cn[i] == -1: a.transitions.add((prev_state, t[0], t[1], '|', -2, i)) else : # counting constraint a.transitions.add((prev_state, t[0], t[1], '|', -2, i, cn[i], cn_i.index(i))) cur_state = a.start # Discover fading states and their overlap states. f = [] # fading states (DFA) f_d = {} # fading states in dictionary, key is string before o = [] # overlap states according fading states (DFA) for i in range(0, len(b)): f_d[b[i]] = [] c_s = cl[i] # c_s is closure states (NFA) for j in range(0, len(r)): if c_s in r[j]: over = list(r[j]) for s in cl: if s in over: over.remove(s) # append only states which have overlap state if over in r: if j not in f: f.append(j) f_d[b[i]].append(j) # append overlap DFA state o.append(r.index(over)) # Remove fading states. for s in f: del a.states[s] # delete deleted states from finite states for s in list(a.final): if s in f: a.final.remove(s) # change numbering for states a_s = sorted(a.states.keys()) # aux. states tmp = {} for k in a_s: tmp[a_s.index(k)] = a.states[k] a.states = tmp # change numbering for final states tmp = set() for s in a.final: tmp.add(a_s.index(s)) a.final = tmp # change numbering for start state a.start = a_s.index(a.start) # Change fading transitions. # transition is tuple of 3 numbers plus history properties # like: (S, A, D) # S is source state # A is alphabet (transition char) # D is destination state # like: (S, A, D, -2, f_i, '|', -4, f_i) # -2 is flag which is needed for execution transition # f_i is flag index # '|' separating needed properties for execution tran. of properties # which will be set after tran. finished # -4 flag which will be set after tran. finish # f_i is flag index # flags: -2 = set, -3 = must be set, -4 = reset, -5 = must be reset #c_s_D = [] # closure state DFA ! #print "cl:",cl #print "r:",r #for c_s_N in cl: #print "c_s_N:",c_s_N #c_s_D.append(r.index([c_s_N])) aux = set() d = {} # transitions for pruning process tmp = [] # helpful list for pruning process tmp_t = [] # helpful list for pruning process for t in a.transitions: t = list(t) # destination state is fading state if t[2] in f: # source state is fading state if t[0] in f: # this transition only if is set flag (=) # append must be set flag for i in range(0, len(b)): if t[0] in f_d[b[i]]: t.append(-3) t.append(i) if cn[i] != -1: # append counting constraint counter 0 t.append(0) t.append(cn_i.index(i)) t.append('|') # change to overlap state t[0] = o[f.index(t[0])] # source state is non fading else : if '|' not in t: t.append('|') # append set flag for i in range(0, len(b)): if t[2] in f_d[b[i]]: t.append(-2) t.append(i) if cn[i] != -1: # append counting constraint set counter t.append(cn[i]) t.append(cn_i.index(i)) # change des. state to overlap state t[2] = o[f.index(t[2])] # source state is fading state elif t[0] in f: # append must be set flag for i in range(0, len(b)): if t[0] in f_d[b[i]]: t.append(-3) t.append(i) if cn[i] != -1: # append counting constraint counter 0 t.append(0) t.append(cn_i.index(i)) t.append('|') # destination state is non-fading state # reset flag for i in range(0, len(b)): if t[0] in f_d[b[i]]: t.append(-4) t.append(i) if cn[i] != -1: # append counting constraint reset counter t.append(0) t.append(cn_i.index(i)) # change to overlap state t[0] = o[f.index(t[0])] # change numbering t[0] = a_s.index(t[0]) t[2] = a_s.index(t[2]) aux.add(tuple(t)) # for pruning process if len(t) == 3: t_a = t + ['|'] else : t_a = t[0:3] + t[t.index('|'):] t_a = tuple(t_a) if t_a in tmp: # select similarly transitions if not d.has_key(t_a): d[t_a] = [tmp_t[tmp.index(t_a)]] if t not in d[t_a]: d[t_a].append(t) else : tmp.append(t_a) tmp_t.append(t) a.transitions = aux # Pruning process (remove similarly transitions). for key in d.keys(): # exist transition without condition if list(key[0:3]) in d[key]: # keep only this transition d[key].remove(list(key[0:3])) for t in d[key]: a.transitions.remove(tuple(t)) else : # find set and remove all her subset c = [] # auxiliary, conditions for tran. c_l = [] # auxiliary, conditions length tran = [] # auxiliary, original tran. for t in d[key]: aux = [] for i in range(3, t.index('|'), 2): aux += [tuple(t[i:i+2])] c.append(aux) c_l.append(len(aux)) tran.append(tuple(t)) while len(c) != 0: s = c[c_l.index(min(c_l))] # set for t_c in list(c): flag = 1 for x in s: if x not in t_c: flag = 0 break # tran. fall into this set if flag and s != t_c: # remove subset tran. del c_l[c.index(t_c)] a.transitions.remove(tran[c.index(t_c)]) c.remove(t_c) # remove set (only from c) del c_l[c.index(s)] c.remove(s) # Append must be reset flag. c = {} # tran. which have must be set flag c_i = {}# indexes of flags n = {} # tran. which have not must be set flag for s in range(0, len(a.states)): c[s] = [] c_i[s] = [] n[s] = [] for t in a.transitions: added = False t = list(t) # must be set tran. if len(t) > 3 and (t.index('|') - 3) > 0: tmp = [] for i in range(3, t.index('|'), 2): if t[i] == -3: tmp.append(t[i+1]) if tmp != []: if tuple(t) not in c[t[0]]: c[t[0]].append(tuple(t)) c_i[t[0]].append(tmp) added = True # not must be set tran. if not added: if tuple(t) not in n[t[0]]: n[t[0]].append(tuple(t)) for s in range(0, len(a.states)): for t in c[s]: # discover tran. char if isinstance(a.alphabet[t[1]], sym_char.b_Sym_char): chars = list(a.alphabet[t[1]].char) else : chars = list(a.alphabet[t[1]].charClass) for t_n in list(n[s]): if isinstance(a.alphabet[t_n[1]], sym_char.b_Sym_char): chars_n = list(a.alphabet[t_n[1]].char) else : chars_n = list(a.alphabet[t_n[1]].charClass) for char in chars_n: if char in chars: # remove bad tran. a.transitions.remove(t_n) n[s].remove(t_n) # append must be reset flag t_n = list(t_n) if '|' not in t_n: for i in c_i[s][c[s].index(t)]: t_n.append('-5') t_n.append(i) t_n.append('|') else : for i in c_i[s][c[s].index(t)]: t_n.insert(3, '-5') t_n.insert(4, i) a.transitions.add(tuple(t_n)) break # Count of flags and counters. self.flag_c = len(b) self.ctr_c = len(cn_i) self.cn_i = cn_i # Add automat flag. a.Flags["History FA"] = True
settings. """ if __name__ == '__main__': # Sidhu Prasana mapping method print("-------------------------------------------------------------------") print(" Example of use: Sidhu-Prasana NFA ") print("-------------------------------------------------------------------") print(" Ruleset: ../../rules/L7/selected.pcre ") print(" Strided: No ") print(" Genereated VHDL output: sidhu_prasana_nfa_impl.vhd ") print("-------------------------------------------------------------------") # Create sindhu_prasana_nfa object cn = sindhu_prasana_nfa.sindhu_prasana_nfa() # Create parser - use default parser Test0 = parser.parser() # Load RE file Test0.load_file("../../rules/L7/selected.pcre") # Parse RE and create NFA cn.create_by_parser(Test0) # Call the compute method cn.compute() # Get number of used LUTs and FFs data = cn.report_logic() print(" Used LUTs estimation: " + str(data[0])) print(" Used FFs estimation: " + str(data[1])) print("-------------------------------------------------------------------") # Save implementation # Open file f = open("sidhu_prasana_nfa_impl.vhd", "w") # Get VHDL code and write the code
def test_compute(self): """compute()""" # Check the correctness of the logical machine output over # self.assertTrue on individual items + focus on the properties # of HistoryFA (transitions, flags, counters) # /abcd/; test with an expression that does not use properties # of HistoryFA par = parser("pcre_parser") par.set_text("/abcd/") history = HistoryFA() history.create_by_parser(par) history.remove_epsilons() NFA = history.get_automaton(True) history.determinise(create_table = True) history.compute(NFA) copy = history.get_automaton() result = nfa_data().load_from_file(aux_func.getPatternMatchDir() + "/algorithms/history_fa/test_data/test_data_1.nfa_data") self.assertTrue(sorted(copy.states.keys()) == sorted(result.states.keys())) self.assertTrue(copy.alphabet == result.alphabet) self.assertTrue(copy.start == result.start) self.assertTrue(copy.final == result.final) self.assertTrue(copy.transitions == result.transitions) # /ab.*cd /; test with an expression that contains .* par = parser("pcre_parser") par.set_text("/ab.*cd/") history = HistoryFA() history.create_by_parser(par) history.remove_epsilons() NFA = history.get_automaton(True) history.determinise(create_table = True) history.compute(NFA) copy = history.get_automaton() result = nfa_data().load_from_file(aux_func.getPatternMatchDir() + "/algorithms/history_fa/test_data/test_data_2.nfa_data") self.assertTrue(sorted(copy.states.keys()) == sorted(result.states.keys())) self.assertTrue(copy.alphabet == result.alphabet) self.assertTrue(copy.start == result.start) self.assertTrue(copy.final == result.final) self.assertTrue(copy.transitions == result.transitions) self.assertTrue(copy.Flags == result.Flags) # /ab[^1234]*cd|efg/; test with an expression containing one # alternation [^1234]*, the second is not par = parser("pcre_parser") par.set_text("/ab[^1234]*cd|efg/") history = HistoryFA() history.create_by_parser(par) history.remove_epsilons() NFA = history.get_automaton(True) history.determinise(create_table = True) history.compute(NFA) copy = history.get_automaton() result = nfa_data().load_from_file(aux_func.getPatternMatchDir() + "/algorithms/history_fa/test_data/test_data_3.nfa_data") self.assertTrue(sorted(copy.states.keys()) == sorted(result.states.keys())) self.assertTrue(copy.alphabet == result.alphabet) self.assertTrue(copy.start == result.start) self.assertTrue(copy.final == result.final) self.assertTrue(copy.transitions == result.transitions) self.assertTrue(copy.Flags == result.Flags)
for ruleset in rulesets: for stride in strides: ruleset_filename = rulesets_prefix + ruleset + rulesets_suffix # remove any non-alphanumeric character ruleset_abc = re.sub('[\W_]', '', ruleset) print('Generating (%s,%d)...' % (ruleset_filename, stride)) start = time.time() # Create clark_nfa object cn = clark_nfa.clark_nfa(True, stride) # Create parser - use default parser Test0 = parser.parser("pcre_parser") # Load RE file Test0.load_file(ruleset_filename) # Parse RE and create NFA cn.create_by_parser(Test0) # Call the compute method cn.compute() # Get number of used LUTs and FFs logic = cn.report_logic() stop = time.time() report.write('%s %d %d %d %d %d %d %d %f\n' % (ruleset_abc, stride, logic[0], logic[1], logic[2], cn.get_state_num(), len(cn.get_set_of_nondeterministic_states()),
# Import used Netbench modules from history_fa import HistoryFA from netbench.pattern_match.parser import parser # EXAMPLE of use for HistoryFA class if __name__ == '__main__': print("-------------------------------------------------------------------") print(" Example of use: History FA ") print("-------------------------------------------------------------------") print(" Ruleset: /ab.*cd/ ") print(" Graphical representation: automaton.dot ") print(" history.dot ") print("-------------------------------------------------------------------") # Create PCRE parser object par = parser("pcre_parser") # Set the ruleset par.set_text("/ab.*cd/") # Create HistoryFA object history = HistoryFA() # Parse the ruleset history.create_by_parser(par) # Remove epsilons history.remove_epsilons() # Get copy of NFA part NFA = history.get_automaton(True) # Determinise the automaton. Record corespondance between DFA states # and NFA states history.determinise(create_table = True) # Create the History FA from deterministic automaton original NFA.