def process_single_ds(uat):
try:
uat_count = 200
# number of total automata to be processed. Make this number smaller to get results faster for a subset
automatas = atma.parse_anml_file(anml_path[uat])
automatas.remove_ors()
automatas = automatas.get_connected_components_as_automatas()
automatas = automatas[:uat_count]
# pick the number of automataon to be processed. comment this line if you want to process the whole benchmark
target_bit_widths = [1, 2, 4, 16]
# bitwidth to be calculated
stats = [[0, 0] for _ in range(len(target_bit_widths))]
for atm in automatas:
b_atm = atma.automata_network.get_bit_automaton(atm, original_bit_width=atm.max_val_dim_bits_len)
# generate bit automaton
for tb_idx, tb in enumerate(target_bit_widths):
if tb == 1:
atm = b_atm.clone()
else:
atm = atma.automata_network.get_strided_automata2(atm=b_atm,
stride_value=tb,
is_scalar=True,
base_value=2,
add_residual=True)
if atm.is_homogeneous is False:
atm.make_homogenous()
minimize_automata(atm)
n_s = atm.nodes_count
n_e = atm.edges_count
stats[tb_idx][0] += n_s
stats[tb_idx][1] += n_e
global_lock.acquire()
print uat
for tb_idx, tb in enumerate(target_bit_widths):
print "bitwidth = ", tb, "number of states = ", stats[tb_idx][0], "number of edges = ", stats[tb_idx][1]
global_lock.release()
return uat, stats
except Exception, e:
tracebackString = traceback.format_exc(e)
print tracebackString
raise StandardError, "\n\nError occurred. Original traceback is\n%s\n" %(tracebackString)
def process_single_ds(uat):
automata_per_stage = 50
exempts = {(AnmalZoo.Snort, 1411)}
automatas = atma.parse_anml_file(anml_path[uat])
automatas.remove_ors()
automatas = automatas.get_connected_components_as_automatas()
uat_count = len(automatas)
automatas = automatas[:uat_count]
uat_count = len(automatas)
number_of_stages = math.ceil(len(automatas) / float(automata_per_stage))
atms_per_stage = int(math.ceil(len(automatas) / float(number_of_stages)))
hdl_folder_name = hd_gen.get_hdl_folder_name(prefix=str(uat), number_of_atms=uat_count,
stride_value=stride, before_match_reg=False,
after_match_reg=False, ste_type=1, use_bram=False,
use_compression=False, compression_depth=-1,
use_mid_fifo=False, use_rst=True)
generator_ins = hd_gen.HDL_Gen(path=os.path.join("/home/gr5yf/FCCM_2020/lut16", hdl_folder_name), before_match_reg=False,
after_match_reg=False, ste_type=1,
total_input_len=automatas[0].max_val_dim_bits_len * pow(2, stride),
bram_shape=None)
for atm_idx, atm in enumerate(automatas):
if (uat, atm_idx) in exempts:
continue
print 'processing {0} stride {3} number {1} from {2}'.format(uat, atm_idx, uat_count, stride)
atm = atm.get_single_stride_graph()
atm = atm.get_single_stride_graph()
atm.make_homogenous()
minimize_automata(atm)
generator_ins.register_automata(atm=atm, use_compression=False, lut_bram_dic={})
if (atm_idx + 1) % atms_per_stage == 0:
generator_ins.register_stage_pending(use_bram=False)
generator_ins.register_stage_pending(use_bram=False)
generator_ins.finilize(dataplane_intcon_max_degree=5, contplane_intcon_max_degree=10)
def process_single_ds(uat):
automatas = atma.parse_anml_file(anml_path[uat])
approximate_final, real_final = [], [] # these lists keep number of reports for each CC
real_states, appr_states = 0, 0 # these integers count number of states
translation_dic = {x: x % approximate_ratio for x in range(automatas.max_val_dim + 1)}
automatas.remove_ors()
automatas = automatas.get_connected_components_as_automatas()
with open(str(uat) + '.txt', "w+") as f:
pass
for atm in automatas[:uat_count]:
real_states+= atm.nodes_count
atm.set_all_symbols_mutation(False)
appr_automata = get_approximate_automata(atm=atm, translation_dic=translation_dic,
max_val_dim=atm.max_val_dim / approximate_ratio)
minimize_automata(automata=appr_automata)
appr_states += appr_automata.nodes_count
run_result = automata_run_stat(atm=atm, file_path=input_path[uat], cycle_detail=True, report_detail=False, bytes_per_dim=1)
real_final.append(run_result[total_reports])
appr_run_result = automata_run_stat(atm=appr_automata, file_path=input_path[uat], cycle_detail=True, report_detail=False, bytes_per_dim=1,
translation_dic=translation_dic)
approximate_final.append(appr_run_result[total_reports])
with open(str(uat) + '.txt', "a") as f:
print >> f, "real reports: " + str(real_final[-1])
print >> f, "approximate reports: " + str(approximate_final[-1])
print >> f, "real nodes count: " + str(atm.nodes_count)
print >> f, "approximate nodes count:" + str(appr_automata.nodes_count)
print >>f, "----------------------------------------------------------------"
with open(str(uat) + '.ttxt', "a") as f:
print >>f, "***************sum*******************"
print >>f, "real reports: " + str(sum(real_final))
print >>f, "approximate reports: " + str(sum(approximate_final))
print >>f, "real nodes count: " + str(real_states)
print >>f, "approximate nodes count:" + str(appr_states)
def process_single_ds(uat):
automatas = atma.parse_anml_file(anml_path[uat])
automatas.remove_ors()
automatas = automatas.get_connected_components_as_automatas()
uat_count = len(automatas)
n_states = 0.0
n_edges = 0.0
total_sym = 0.0
for atm_idx, atm in enumerate(automatas):
minimize_automata(atm)
all_nodes = filter(lambda n: n.id != 0, atm.nodes) # filter fake root
all_nodes_symbols_len_count = [
len(list(n.symbols.points)) for n in all_nodes
]
n_s = atm.nodes_count
n_states += n_s
n_e = atm.edges_count
n_edges += n_e
t_s = sum(all_nodes_symbols_len_count)
total_sym += t_s
global_lock.acquire()
print str(uat), "\t nodes count = ", n_states, "\tedges count = ", n_edges, "\tAvg node degree = ",\
2 * n_edges / n_states, "\tAvg symbol size = ", total_sym/n_states
global_lock.release()
return True
orig_atm_nodes_count = []
fst_stride_nodes_count = []
sec_stride_nodes_count = []
third_stride_nodes_count = []
orig_automatas = automata.get_connected_components_as_automatas()
for orig_cc in tqdm(orig_automatas, unit="automata"):
orig_atm_nodes_count.append((orig_cc.get_number_of_nodes(), orig_cc.edges_count,
orig_cc.max_STE_out_degree(),orig_cc.max_STE_in_degree() ))
orig_cc.remove_all_start_nodes()
fst_st_atm = orig_cc.get_single_stride_graph()
fst_st_atm.make_homogenous()
minimize_automata(fst_st_atm)
fst_stride_nodes_count.append((fst_st_atm.get_number_of_nodes(), fst_st_atm.edges_count,
fst_st_atm.max_STE_out_degree(),fst_st_atm.max_STE_in_degree()))
sec_st_atm = fst_st_atm.get_single_stride_graph()
sec_st_atm.make_homogenous()
minimize_automata(sec_st_atm)
sec_stride_nodes_count.append((sec_st_atm.get_number_of_nodes(), sec_st_atm.edges_count,
sec_st_atm.max_STE_out_degree(),sec_st_atm.max_STE_in_degree()))
thd_st_atm = sec_st_atm.get_single_stride_graph()
thd_st_atm.make_homogenous()
minimize_automata(thd_st_atm)
third_stride_nodes_count.append((thd_st_atm.get_number_of_nodes(), thd_st_atm.edges_count,
thd_st_atm.max_STE_out_degree(),thd_st_atm.max_STE_in_degree()))
bc_sym_dict = get_equivalent_symbols([atm], replace=True)
translation_list = []
print atm.get_summary(logo="9bit atm info")
atm = atm.get_single_stride_graph()
print atm.get_summary(logo="done st1")
atm = atm.get_single_stride_graph()
print atm.get_summary(logo="done st2")
atm = atm.get_single_stride_graph()
print atm.get_summary(logo="done st3")
if atm.is_homogeneous is False:
atm.make_homogenous()
minimize_automata(atm)
print atm.get_summary()
strided_automatas.append(atm.id)
generator_ins.register_automata(atm=atm, use_compression=use_compression, byte_trans_map=bc_sym_dict if use_compression else None,
translation_list=translation_list)
if use_compression:
generator_ins.register_compressor([atm.id], byte_trans_map=bc_sym_dict,
translation_list=translation_list)
if (atm_idx + 1) % atms_per_stage == 0:
generator_ins.register_stage_pending()
generator_ins.register_stage_pending()
for stride in range(max_stride,
max_stride + 1): # one more for the original automata
acc_switch_map = np.zeros((switch_size, switch_size))
print "starting stride ", stride
for cc_idx, cc in enumerate(ccs[:]):
print "processing {} , id {}".format(anml, cc_idx)
print cc.get_summary(logo="original")
for _ in range(stride):
cc = cc.get_single_stride_graph()
if cc.is_homogeneous is False:
cc.make_homogenous()
utility.minimize_automata(cc)
cc.fix_split_all()
print cc.get_summary()
bfs_cost, bfs_label_dictionary = cc.bfs_rout(routing_template)
switch_map = cc.get_connectivity_matrix(
node_dictionary=bfs_label_dictionary)
acc_mat_size, _ = acc_switch_map.shape
new_mat_size, _ = switch_map.shape
if new_mat_size > acc_mat_size:
switch_map[:acc_mat_size, :acc_mat_size] += acc_switch_map
acc_switch_map = switch_map
def process_single_ds(uat):
return_result = {}
#result_dir = out_dir_prefix + str(uat)
#shutil.rmtree(result_dir, ignore_errors=True)
#os.mkdir(result_dir)
# cleaning the result folder
automata_per_stage = 100
# this is a pipelineing parameter for staging as pipeline. We usually use 50 for this parameter
automatas = atma.parse_anml_file(anml_path[uat])
automatas.remove_ors()
automatas = automatas.get_connected_components_as_automatas()
#uat_count = 10 # number of automata to be processed
uat_count = len(
automatas
) # comment this to test a subset of automatons defined in uat_count
automatas = automatas[:uat_count]
uat_count = len(automatas)
number_of_stages = int(
math.ceil(len(automatas) / float(automata_per_stage)))
# number of pipleine stages
atms_per_stage = int(math.ceil(len(automatas) / float(number_of_stages)))
hdl_folder_name = hd_gen.get_hdl_folder_name(prefix=str(uat),
number_of_atms=len(automatas),
stride_value=0,
before_match_reg=False,
after_match_reg=False,
ste_type=1,
use_bram=True,
use_compression=False,
compression_depth=-1,
use_mid_fifo=use_mid_fifo,
use_rst=use_rst)
print "folder name to store the HDLs:", hdl_folder_name
generator_ins = hd_gen.HDL_Gen(path=os.path.join(
"/home/gr5yf/FCCM_2020/bram8", hdl_folder_name),
before_match_reg=False,
after_match_reg=False,
ste_type=1,
total_input_len=dbw,
use_mid_fifo=use_mid_fifo,
use_rst=use_rst,
bram_shape=(512, 36))
for atm_idx, atm in enumerate(automatas):
print 'processing {0} number {1} from {2}'.format(
uat, atm_idx + 1, uat_count)
minimize_automata(atm)
lut_bram_dic = {
n: tuple((2 for _ in range(atm.stride_value)))
for n in atm.nodes if n.is_fake is False
}
generator_ins.register_automata(atm=atm,
use_compression=False,
lut_bram_dic=lut_bram_dic)
if (atm_idx + 1) % atms_per_stage == 0:
generator_ins.register_stage_pending(use_bram=True)
generator_ins.register_stage_pending(use_bram=True)
generator_ins.finilize(dataplane_intcon_max_degree=5,
contplane_intcon_max_degree=10)
return uat, return_result
def process_anml(bitwidth, input_directory, automata_per_stage):
# This is the directory name to be created for HDL files
output_hdl_directory = input_directory + '/' + str(bitwidth) + '_' + str(
automata_per_stage)
anml_input_files = glob.glob(input_directory + '/*.anml')
print("ANML Files: ", anml_input_files)
# Clean up directory
shutil.rmtree(output_hdl_directory, ignore_errors=True)
os.mkdir(output_hdl_directory)
SYMBOLIC = True
# Create a directory name for the HDL code
hdl_folder_name = hd_gen.get_hdl_folder_name(
prefix=output_hdl_directory,
number_of_atms=len(anml_input_files),
stride_value=0,
before_match_reg=False,
after_match_reg=False,
ste_type=1,
use_bram=False,
use_compression=False,
compression_depth=-1,
symbolic=SYMBOLIC)
print("Folder name to store the HDLs: ", hdl_folder_name)
# Create a hardware Generator
generator_ins = hd_gen.HDL_Gen(path=hdl_folder_name,
before_match_reg=False,
after_match_reg=False,
ste_type=1,
total_input_len=dbw,
symbolic=SYMBOLIC)
# Iterate through the ANML files in the directory
for index, anml_input_file in enumerate(anml_input_files):
print("Parsing Automata: {}".format(anml_input_file))
module_name = 'Automata_tt_' + str(index)
verilog_filename = hdl_folder_name + '/automata_tt_' + str(
index) + '.sv'
# Parse the ANML file
automata = atma.parse_anml_file(anml_input_file)
# Minimizing the automata with NFA heuristics
if minimize:
print("Minimizing Automata: {}".format(anml_input_file))
minimize_automata(automata)
#atma.generate_anml_file(anml_input_file + "_min.anml", automata)
else:
print("No minimization of Automata")
tables, start_states, accept_states = atma.generate_tt(automata)
inputs = atma.build_truthtable(tables, start_states, accept_states,
module_name, verilog_filename)
print "Inputs: ", inputs
# for now, we will use this automata proxy
automata = Automatanetwork('tt_' + str(index),
True,
1,
255,
inputs=inputs)
new_node = S_T_E(start_type=StartType.non_start,
is_report=True,
is_marked=False,
id=automata.get_new_id(),
symbol_set=None,
adjacent_S_T_E_s=None,
report_residual=1,
report_code=1)
automata.add_element(new_node)
print "Number of states: ", automata.nodes_count
# Register this automaton
generator_ins.register_automata(atm=automata, use_compression=False)
# We've got another batch of automata_per_stage automata to stage
if (index + 1) % automata_per_stage == 0:
generator_ins.register_stage_pending(single_out=False,
use_bram=False)
# DO we need this? maybe if our number of automata is not a perfect multiple
# of automata_per_stage?
generator_ins.register_stage_pending(single_out=False, use_bram=False)
#Finalize and wrap up HDL in archive folder
generator_ins.finilize()
# Using gztar to handle LARGE automata workloads
shutil.make_archive(hdl_folder_name, 'gztar', output_hdl_directory)
shutil.rmtree(output_hdl_directory)
uat = AnmalZoo.Snort
automatas = atma.parse_anml_file(anml_path[uat])
automatas.remove_ors()
automatas = automatas.get_connected_components_as_automatas()
for atm in automatas[7:8]:
atm.draw_graph('orig.svg')
#trad_s1 = atm.get_single_stride_graph()
trad_s2 = atm.get_single_stride_graph()
trad_s2.draw_graph('trad_before_homo.svg')
trad_s2.make_homogenous(use_espresso=False)
#minimize_automata(trad_s2)
trad_s2.fix_split_all()
trad_s2.draw_graph('trad_before.svg')
minimize_automata(trad_s2, combine_equal_syms_only=True)
trad_s2.draw_graph('trad.svg')
#new_s1 = atm.get_single_stride_graph()
new_s2 = atm.get_single_stride_graph()
new_s2.draw_graph('new_before_homo.svg')
new_s2.make_homogenous(use_espresso=True)
new_s2.draw_graph('new_before.svg')
minimize_automata(new_s2, combine_equal_syms_only=True, remove_dead_states=False)
new_s2.draw_graph('new.svg')
print "trad has {} nodes new has {} nodes".format(trad_s2.nodes_count, new_s2.nodes_count)
print "trad has {} edges new has {} edges".format(trad_s2.edges_count, new_s2.edges_count)
#exit(0)
def process_single_ds(uat):
return_result = {}
result_dir = out_dir + str(uat)
shutil.rmtree(result_dir, ignore_errors=True)
os.mkdir(result_dir)
exempts = {(AnmalZoo.Snort, 1411)}
max_target_stride = 2
uat_count = 200
automata_per_stage = 50
automatas = atma.parse_anml_file(anml_path[uat])
automatas.remove_ors()
automatas = automatas.get_connected_components_as_automatas()
after_match_reg = False
actual_bram = False # if True, actual bram will be used. Otherwise, LUT emulates bram
automatas = automatas[:uat_count]
uat_count = len(automatas)
number_of_stages = math.ceil(len(automatas) / float(automata_per_stage))
atms_per_stage = int(math.ceil(len(automatas) / float(number_of_stages)))
for hom_between, is_Bram in [(False, True)]:
for stride_val in range(max_target_stride + 1):
hdl_folder_name = hd_gen.get_hdl_folder_name(
prefix=str(uat),
number_of_atms=len(automatas),
stride_value=stride_val,
before_match_reg=False,
after_match_reg=after_match_reg,
ste_type=1,
use_bram=is_Bram,
use_compression=False,
compression_depth=-1)
generator_ins = hd_gen.HDL_Gen(
path=os.path.join(result_dir, hdl_folder_name),
before_match_reg=False,
after_match_reg=False,
ste_type=1,
total_input_len=automatas[0].max_val_dim_bits_len *
pow(2, stride_val),
bram_shape=(512, 36))
for atm_idx, atm in enumerate(automatas):
if (uat, atm_idx) in exempts:
continue
print 'processing {0} stride {3} number {1} from {2}'.format(
uat, atm_idx, uat_count, stride_val)
for _ in range(stride_val):
if is_Bram is True and hom_between is True and atm.is_homogeneous is False:
atm.make_homogenous()
atm.make_parentbased_homogeneous()
atm = atm.get_single_stride_graph()
if atm.is_homogeneous is False:
atm.make_homogenous()
minimize_automata(atm)
if is_Bram is True and hom_between is False:
atm.fix_split_all()
if is_Bram:
lut_bram_dic = {
n: tuple((2 for _ in range(atm.stride_value)))
for n in atm.nodes if n.is_fake is False
}
else:
lut_bram_dic = {}
generator_ins.register_automata(atm=atm,
use_compression=False,
lut_bram_dic=lut_bram_dic)
if (atm_idx + 1) % atms_per_stage == 0:
generator_ins.register_stage_pending(single_out=False,
use_bram=actual_bram)
generator_ins.finilize()
return uat, return_result
#Snort, EntityResolution, ClamAV, Hamming, Dotstart, Custom, Bro217, Levenstein, Bril,
# Randomfor, Dotstar03, ExactMath,Dotstar06, Fermi, PowerEN, Protomata, Dotstart09, Ranges1, SPM, Ranges 05
#SynthBring, Synthcorering
uat = AnmalZoo.Hamming
automatas = atma.parse_anml_file(anml_path[uat])
automata_name = str(uat)
exempts = {(AnmalZoo.Snort, 1411)}
automatas.remove_ors()
automatas = automatas.get_connected_components_as_automatas()
print("Number of automata: ", len(automatas))
print(automata_name)
filed_names = ['#States', '#Edges', 'max_fan_in', 'max_fan_out', 'total_sym']
for atm_idx, atm in enumerate(automatas):
print "Processing:", uat, " ", atm_idx
atm1E = atm.get_single_stride_graph()
atm2E = atm1E.get_single_stride_graph()
atm2E.make_homogenous(plus_src=False)
minimize_automata(atm2E)
atm2E.fix_split_all()
print "HI"
atm.draw_graph("original.svg")
# draw the autoamton to an svg file
b_atm = atma.automata_network.get_bit_automaton(atm, original_bit_width=atm.max_val_dim_bits_len)
#binary automata
print b_atm.get_summary(logo=" of bitwise automata")
# print information about the binary automata
new_bw_atm = atma.automata_network.get_strided_automata2(atm=b_atm,
stride_value=dbw, is_scalar=True, base_value=2,
add_residual=True)
# new automata that has dbw bits per symbol
new_bw_atm.make_homogenous()
# make the autoamta homogeneous
minimize_automata(new_bw_atm)
# minimizing the new autoamata
print new_bw_atm.get_summary(logo=" of %d-bit automata"%(dbw,))
# print summary of the new autoamta
new_bw_atm.draw_graph("new_bw_atm.svg")
print "SVG files were generated."
uat_count = 200
automatas = atma.parse_anml_file(anml_path[uat])
automatas.remove_ors()
automatas = automatas.get_connected_components_as_automatas()
atm = automatas[uat_index]
atm.draw_graph("original.svg")
#eight_stride_1 = atm.get_single_stride_graph()
#eight_stride_1.make_homogenous()
#minimize_automata(eight_stride_1)
#eight_stride_1.draw_graph("eight_bit_strie1.svg")
b_atm = atma.automata_network.get_bit_automaton(
atm, original_bit_width=atm.max_val_dim_bits_len)
fourbit_atm = atma.automata_network.get_strided_automata2(atm=b_atm,
stride_value=4,
is_scalar=True,
base_value=2,
add_residual=True)
four_bit_s1 = fourbit_atm.get_single_stride_graph()
four_bit_s2 = four_bit_s1.get_single_stride_graph()
four_bit_s2.make_homogenous()
minimize_automata(four_bit_s2)
print "four bit=", four_bit_s2.nodes_count
four_bit_s2.draw_graph("fourbit_s2.svg")
def thread_func(ds):
try:
full_atm = atma.parse_anml_file(anml_path[ds])
full_atm.remove_ors()
atms_list = full_atm.get_connected_components_as_automatas()
atms_list = atms_list[:atms_count]
with open(str(ds) + '4bit.csv', 'w') as csv_file:
csv_writer = csv.writer(csv_file,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
csv_writer.writerow(filed_names)
for curr_atm in atms_list:
b_atm = atma.automata_network.get_bit_automaton(
curr_atm, original_bit_width=curr_atm.max_val_dim_bits_len)
four_b_atm = atma.automata_network.get_strided_automata2(
atm=b_atm,
stride_value=4,
is_scalar=True,
base_value=2,
add_residual=True)
curr_atm = four_b_atm.get_single_stride_graph()
curr_atm.make_homogenous()
minimize_automata(curr_atm)
curr_atm.fix_split_all()
match_graph = nx.Graph()
parent_sym_to_product_sym_dic = {}
for ste in curr_atm.nodes:
if ste.is_fake:
continue # fake root is not considered
match_graph.add_node(ste)
# here we find the union of parents and find the version with false posetive versions
comb_sym_set = PackedIntervalSet(
[]
) # create an empty symbol sets to find union of parents symbol set
for pred in curr_atm.get_predecessors(ste):
if pred.is_fake:
continue
for ivl in pred.symbols.intervals:
comb_sym_set.add_interval(
ivl
) # finding the union of parents symbol sets
# making symbol set smaller
comb_sym_set.prone()
comb_sym_set.merge()
comb_sym_set = comb_sym_set.get_combinatorial_symbol_set()
parent_sym_to_product_sym_dic[ste] = comb_sym_set
# now having a graph with no edge, we add edge between nodes if they can be combined with each other
# Two nodes can be combined if these two conditions are satisfied. First, they should not have a common symbol
# second, their parrents should also not have common symbols
for n in match_graph.nodes():
if n == ste:
continue # we do not check a node with itself
if is_there_common_sym(ste.symbols, n.symbols):
continue # first condition has not been met
elif is_there_common_sym(
parent_sym_to_product_sym_dic[ste],
parent_sym_to_product_sym_dic[n]):
continue # second condition has not been met
else: # both condiotions have been met, we can create an edge between ste, n
match_graph.add_edge(ste, n)
matching_result = matching_alg.max_weight_matching(match_graph)
print "number of orignal nodes ", curr_atm.nodes_count
print "number of matching nodes ", 2 * len(matching_result)
csv_writer.writerow(
[curr_atm.nodes_count, 2 * len(matching_result)])
except Exception as ex:
print traceback.print_exc()
my_Automata.add_element(ste1)
my_Automata.add_element(ste2)
my_Automata.add_element(ste3)
my_Automata.add_element(ste4)
my_Automata.add_element(ste5)
my_Automata.add_element(ste6)
my_Automata.add_element(ste7)
my_Automata.add_element(ste8)
my_Automata.add_edge(ste1, ste2)
my_Automata.add_edge(ste2, ste3)
my_Automata.add_edge(ste3, ste4)
my_Automata.add_edge(ste4, ste5)
my_Automata.add_edge(ste5, ste6)
my_Automata.add_edge(ste6, ste7)
my_Automata.add_edge(ste7, ste8)
my_Automata.draw_graph('t.svg')
print my_Automata.get_summary()
my_Automata = my_Automata.get_single_stride_graph()
my_Automata = my_Automata.get_single_stride_graph()
my_Automata.make_homogenous()
minimize_automata(my_Automata)
my_Automata.fix_split_all()
print my_Automata.get_summary()
my_Automata.draw_graph('s.svg')
def process_single_ds(uat):
try:
#uat = AnmalZoo.Ranges05
return_result = {}
result_dir = out_dir + str(uat)
shutil.rmtree(result_dir, ignore_errors=True)
os.mkdir(result_dir)
exempts = {(AnmalZoo.Snort, 1411)}
min_target_stride, max_target_stride = 3, 3
uat_count = 200
automatas = atma.parse_anml_file(anml_path[uat])
automatas.remove_ors()
automatas = automatas.get_connected_components_as_automatas()
#uat_count = len(automatas) # comment this to test a subset of automatons defined in uat_count
automatas = automatas[:uat_count]
uat_count = len(automatas)
filed_names = [
'number_of_states', 'number_of_edges', 'max_fan_in', 'max_fan_out',
'max_symbol_len', 'min_symbol_len', 'total_sym'
]
for hom_between, is_Bram in [(False, True)]:
n_states = [0.0 for _ in range(max_target_stride + 1)]
n_edges = [0.0 for _ in range(max_target_stride + 1)]
max_fan_in = [0.0 for _ in range(max_target_stride + 1)]
max_fan_out = [0.0 for _ in range(max_target_stride + 1)]
max_sym_len = [0.0 for _ in range(max_target_stride + 1)]
min_sym_len = [0.0 for _ in range(max_target_stride + 1)]
total_sym = [0.0 for _ in range(max_target_stride + 1)]
csv_writers = []
for i in range(max_target_stride + 1):
f = open(
result_dir + '/S' + str(i) + '_' + str(uat_count) +
'is_HNH' + str(hom_between) + 'is_Bram' + str(is_Bram) +
'len' + str(uat_count) + '.csv', 'w')
csv_writer = csv.writer(f,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
csv_writer.writerow(filed_names)
csv_writers.append(csv_writer)
for atm_idx, atm in enumerate(automatas):
b_atm = atma.automata_network.get_bit_automaton(
atm, original_bit_width=atm.max_val_dim_bits_len)
atm = atma.automata_network.get_strided_automata2(
atm=b_atm,
stride_value=4,
is_scalar=True,
base_value=2,
add_residual=True)
for stride_val in reversed(
range(min_target_stride, max_target_stride + 1)):
if (uat, atm_idx) in exempts:
continue
print 'processing {0} stride {3} number {1} from {2}'.format(
uat, atm_idx, uat_count, stride_val)
s_atm = atm
for _ in range(stride_val):
if s_atm is atm:
s_atm = atm.get_single_stride_graph()
else:
s_atm = s_atm.get_single_stride_graph()
if s_atm.is_homogeneous is False:
s_atm.make_homogenous()
minimize_automata(s_atm)
if is_Bram is True and hom_between is False:
s_atm.fix_split_all()
all_nodes = filter(lambda n: n.id != 0,
s_atm.nodes) # filter fake root
all_nodes_symbols_len_count = [
len(n.symbols) for n in all_nodes
]
n_s = s_atm.nodes_count
n_states[stride_val] += n_s
n_e = s_atm.edges_count
n_edges[stride_val] += n_e
m_f_i = s_atm.max_STE_in_degree()
max_fan_in[stride_val] += m_f_i
m_f_o = s_atm.max_STE_out_degree()
max_fan_out[stride_val] += m_f_o
mx_s_l = max(all_nodes_symbols_len_count)
max_sym_len[stride_val] += mx_s_l
mn_s_l = min(all_nodes_symbols_len_count)
min_sym_len[stride_val] += mn_s_l
t_s = sum(all_nodes_symbols_len_count)
total_sym[stride_val] += t_s
csv_writers[stride_val].writerow(
[n_s, n_e, m_f_i, m_f_o, mx_s_l, mn_s_l, t_s])
del csv_writers
for i in range(max_target_stride + 1):
n_states[i] /= uat_count
n_edges[i] /= uat_count
max_fan_in[i] /= uat_count
max_fan_out[i] /= uat_count
max_sym_len[i] /= uat_count
min_sym_len[i] /= uat_count
total_sym[i] /= uat_count
return_result[(is_Bram, i)] = (n_states, n_edges, max_fan_in,
max_fan_out, max_sym_len,
min_sym_len, total_sym)
with open(out_dir + 'summary.txt', 'a+') as f:
fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
to_w_lns = []
to_w_lns.append(
str(uat) + "L" + str(uat_count) + "S" + str(i) +
"BRam" + str(is_Bram) + "\n")
to_w_lns.append(" average number of states = " +
str(n_states[i]) + "\n")
to_w_lns.append(" average number of edges = " +
str(n_edges[i]) + "\n")
to_w_lns.append(" average max fan-in = " +
str(max_fan_in[i]) + "\n")
to_w_lns.append(" average max fan-out = " +
str(max_fan_out[i]) + "\n")
to_w_lns.append(" average max sym-len = " +
str(max_sym_len[i]) + "\n")
to_w_lns.append(" average min sym-len = " +
str(min_sym_len[i]) + "\n")
to_w_lns.append(" average total_sym-len = " +
str(total_sym[i]) + "\n")
f.writelines(to_w_lns)
fcntl.flock(f, fcntl.LOCK_UN)
return uat, return_result
except Exception, e:
tracebackString = traceback.format_exc(e)
print tracebackString
raise StandardError, "\n\nError occurred. Original traceback is\n%s\n" % (
tracebackString)
def process_anml(bitwidth, input_directory, automata_per_stage):
# This is the directory name to be created for HDL files
output_hdl_directory = input_directory + '/' + str(bitwidth) + '_' + str(
automata_per_stage)
anml_input_files = glob.glob(input_directory + '/*.anml')
# Clean up directory
shutil.rmtree(output_hdl_directory, ignore_errors=True)
os.mkdir(output_hdl_directory)
# Create a directory name for the HDL code
hdl_folder_name = hd_gen.get_hdl_folder_name(
prefix=output_hdl_directory,
number_of_atms=len(anml_input_files),
stride_value=0,
before_match_reg=False,
after_match_reg=False,
ste_type=1,
use_bram=False,
use_compression=False,
compression_depth=-1,
symbolic=False)
# Create a hardware Generator
generator_ins = hd_gen.HDL_Gen(path=hdl_folder_name,
before_match_reg=False,
after_match_reg=False,
ste_type=1,
total_input_len=dbw,
symbolic=False)
# Iterate through the ANML files in the directory
for index, anml_input_file in enumerate(anml_input_files):
# Grab the automata file number
automata_number = re.search('\d+', anml_input_file).group(0)
# Parse the ANML file
automata = atma.parse_anml_file(anml_input_file)
if dbw == 16:
print "Doing 16-bit!"
automata_with_set_bw = automata.get_single_stride_graph()
else:
print "Doing 8-bit!"
automata_with_set_bw = automata
assert dbw == automata_with_set_bw.total_bits_len, "Bitwidth assumption is incorrect!"
if not automata_with_set_bw.is_homogeneous:
print "Converting to homogeneous automaton"
automata_with_set_bw.make_homogenous()
# Minimizing the automata with NFA heuristics
if minimize:
minimize_automata(automata_with_set_bw)
#atma.generate_anml_file(anml_input_file + "_min.anml", automata)
else:
print("No minimization of Automata")
# Drawing automata graph
if drawing:
print "Drawing automata svg graph"
automata_with_set_bw.draw_graph(anml_input_file +
"_minimized_hw.svg")
automata_info.append('{},{},{}\n'.format(
automata_number, str(automata_with_set_bw.nodes_count),
str(automata_with_set_bw.edges_count)))
# # Register this automaton
generator_ins.register_automata(atm=automata_with_set_bw,
use_compression=False)
# We've got another batch of automata_per_stage automata to stage
if (index + 1) % automata_per_stage == 0:
generator_ins.register_stage_pending(single_out=False,
use_bram=False)
# DO we need this? maybe if our number of automata is not a perfect multiple
# of automata_per_stage?
generator_ins.register_stage_pending(single_out=False, use_bram=False)
#Finalize and wrap up HDL in archive folder
generator_ins.finilize()
# Using gztar to handle LARGE automata workloads
shutil.make_archive(hdl_folder_name, 'gztar', output_hdl_directory)
shutil.rmtree(output_hdl_directory)
# Write the automata node and edge count to a file
with open(hdl_folder_name + '.stats', 'w') as output_file:
output_file.write("Number of States, Number of Edges\n")
output_file.write("---------------------------------\n")
for automata_string in automata_info:
output_file.write(automata_string)
def process_single_ds(uat):
all_automata = atma.parse_anml_file(anml_path[uat])
all_automata.remove_ors()
automatas = all_automata.get_connected_components_as_automatas()
if len(automatas) > number_of_autoamtas:
#automatas = random.sample(automatas, number_of_autoamtas)
automatas = automatas[:number_of_autoamtas]
number_of_stages = math.ceil(len(automatas) / float(automata_per_stage))
atms_per_stage = int(math.ceil(len(automatas) / float(number_of_stages)))
for stride_val in range(1):
hdl_apth = hd_gen.get_hdl_folder_name(
prefix="comptest" + str(uat),
number_of_atms=len(automatas),
stride_value=stride_val,
before_match_reg=before_match_reg,
after_match_reg=after_match_reg,
ste_type=ste_type,
use_bram=use_bram,
use_compression=use_compression,
compression_depth=compression_depth)
generator_ins = hd_gen.HDL_Gen(
path=hdl_apth,
before_match_reg=before_match_reg,
after_match_reg=after_match_reg,
ste_type=ste_type,
total_input_len=hd_gen.HDL_Gen.get_bit_len(
all_automata.max_val_dim) * pow(2, stride_val))
for atm_idx, atm in enumerate(automatas):
if (uat, atm_idx) in exempts:
continue
print 'processing {0} stride{3} automata {1} from {2}'.format(
uat, atm_idx + 1, len(automatas), stride_val)
bc_bits_len = 8
if use_compression:
bc_sym_dict = get_equivalent_symbols([atm], replace=True)
bc_bits_len = int(
math.ceil(math.log(max(bc_sym_dict.values()), 2)))
translation_list = []
for s in range(stride_val):
atm = atm.get_single_stride_graph()
if use_compression and s < compression_depth:
new_translation = get_equivalent_symbols([atm],
replace=True)
translation_list.append(new_translation)
if atm.is_homogeneous is False:
atm.make_homogenous()
minimize_automata(atm)
#lut_bram_dic = {n: (1, 2) for n in atm.nodes}
generator_ins.register_automata(
atm=atm,
use_compression=use_compression,
byte_trans_map=bc_sym_dict if use_compression else None)
if use_compression:
generator_ins.register_compressor(
[atm.id],
byte_trans_map=bc_sym_dict,
translation_list=translation_list)
if (atm_idx + 1) % atms_per_stage == 0:
generator_ins.register_stage_pending(use_bram=use_bram)
generator_ins.finilize()
def process_single_ds(uat):
#uat = AnmalZoo.Ranges05
return_result = {}
result_dir = out_dir + str(uat)
shutil.rmtree(result_dir, ignore_errors=True)
os.mkdir(result_dir)
exempts = {(AnmalZoo.Snort, 1411)}
max_target_stride = 2
uat_count = 200
automata_per_stage = 50
automatas = atma.parse_anml_file(anml_path[uat])
automatas.remove_ors()
automatas = automatas.get_connected_components_as_automatas()
after_match_reg = False
actual_bram = False # if True, actual bram will be used. Otherwise, LUT emulates bram
#uat_count = len(automatas) # comment this to test a subset of automatons defined in uat_count
automatas = automatas[:uat_count]
uat_count = len(automatas)
number_of_stages = math.ceil(len(automatas) / float(automata_per_stage))
atms_per_stage = int(math.ceil(len(automatas) / float(number_of_stages)))
for hom_between, is_Bram in [(False, True)]:
hdl_writers = []
for i in range(max_target_stride + 1):
hdl_folder_name = hd_gen.get_hdl_folder_name(prefix=str(uat), number_of_atms=len(automatas),
stride_value=i, before_match_reg=False,
after_match_reg=after_match_reg, ste_type=1, use_bram=is_Bram,
use_compression=False, compression_depth=-1)
generator_ins = hd_gen.HDL_Gen(path=os.path.join(result_dir, hdl_folder_name), before_match_reg=False,
after_match_reg=after_match_reg, ste_type=1,
total_input_len=4 * pow(2, i),
bram_shape=(512, 36))
hdl_writers.append(generator_ins)
for atm_idx, atm in enumerate(automatas):
b_atm = atma.automata_network.get_bit_automaton(atm, original_bit_width=atm.max_val_dim_bits_len)
atm = atma.automata_network.get_strided_automata2(atm=b_atm,
stride_value=4,
is_scalar=True,
base_value=2,
add_residual=True)
for stride_val in reversed(range(max_target_stride + 1)):
if (uat, atm_idx) in exempts:
continue
print 'processing {0} stride {3} number {1} from {2}'.format(uat, atm_idx, uat_count, stride_val)
s_atm = atm
for _ in range(stride_val):
if s_atm is atm:
s_atm = atm.get_single_stride_graph()
else:
s_atm = s_atm.get_single_stride_graph()
if s_atm.is_homogeneous is False:
s_atm.make_homogenous()
minimize_automata(s_atm)
if is_Bram is True and hom_between is False:
s_atm.fix_split_all()
if is_Bram:
lut_bram_dic = {n: tuple((2 for _ in range(s_atm.stride_value))) for n in s_atm.nodes if
n.is_fake is False}
else:
lut_bram_dic = {}
hdl_writers[stride_val].register_automata(atm=s_atm, use_compression=False, lut_bram_dic=lut_bram_dic)
if (atm_idx + 1) % atms_per_stage == 0:
hdl_writers[stride_val].register_stage_pending(single_out=False, use_bram=actual_bram)
for i in range(max_target_stride + 1):
hdl_writers[i].finilize()
return uat, return_result
draw_individually = False
report_locs_G4 = set([128 + (i * 256) + j for i in range(4) for j in range(8)])
for anml in [AnmalZoo.EntityResolution]:
automata = atma.parse_anml_file(anml_path[anml])
automata.remove_ors()
#utility.minimize_automata(automata)
# accumulative switch map
ccs = automata.get_connected_components_as_automatas()
residual_atms, same_incon_list, curr_node_count, residual_node_count = [], [], 0, 0
for atm_idx, atm in enumerate(ccs):
utility.minimize_automata(atm)
atm.remove_all_start_nodes()
atm1 = atma.automata_network.get_bit_automaton(atm,
original_bit_width=8)
print "finished bitautomata for %d" % (atm_idx)
atm4 = atma.automata_network.get_strided_automata2(atm=atm1,
stride_value=4,
is_scalar=True,
base_value=2,
add_residual=True)
atm8 = atm4.get_single_stride_graph()
atm8.make_homogenous()
print "finished homogeneous for %d" % (atm_idx)
def process_single_ds(uat):
exempts = {(AnmalZoo.Snort, 1411)}
automata_per_stage = 50
# this is a pipelineing parameter for staging as pipeline. We usually use 50 for this parameter
automatas = atma.parse_anml_file(anml_path[uat])
automatas.remove_ors()
automatas = automatas.get_connected_components_as_automatas()
#uat_count = 10 # number of automata to be processed
uat_count = len(
automatas
) # comment this to test a subset of automatons defined in uat_count
automatas = automatas[:uat_count]
uat_count = len(automatas)
number_of_stages = int(
math.ceil(len(automatas) / float(automata_per_stage)))
# number of pipleine stages
atms_per_stage = int(math.ceil(len(automatas) / float(number_of_stages)))
hdl_folder_name = hd_gen.get_hdl_folder_name(prefix=str(uat),
number_of_atms=len(automatas),
stride_value=stride,
before_match_reg=False,
after_match_reg=False,
ste_type=1,
use_bram=True,
use_compression=False,
compression_depth=-1,
use_mid_fifo=use_mid_fifo,
use_rst=use_rst)
print "folder name to store the HDLs:", hdl_folder_name
generator_ins = hd_gen.HDL_Gen(
path=os.path.join("/home/gr5yf/FCCM_2020/bram16", hdl_folder_name),
before_match_reg=False,
after_match_reg=False,
ste_type=1,
total_input_len=automatas[0].max_val_dim_bits_len * pow(2, stride),
use_mid_fifo=use_mid_fifo,
use_rst=use_rst,
bram_shape=(512, 36))
for atm_idx, atm in enumerate(automatas):
if (uat, atm_idx) in exempts:
continue
print 'processing {0} number {1} from {2}'.format(
uat, atm_idx + 1, uat_count)
for s in range(stride):
atm = atm.get_single_stride_graph()
atm.make_homogenous()
minimize_automata(atm)
atm.fix_split_all()
lut_bram_dic = {
n: tuple((2 for _ in range(atm.stride_value)))
for n in atm.nodes if n.is_fake is False
}
generator_ins.register_automata(atm=atm,
use_compression=False,
lut_bram_dic=lut_bram_dic)
if (atm_idx + 1) % atms_per_stage == 0:
generator_ins.register_stage_pending(use_bram=True)
generator_ins.register_stage_pending(use_bram=True)
generator_ins.finilize(dataplane_intcon_max_degree=5,
contplane_intcon_max_degree=10)
return
org_atm.draw_graph('original.svg')
bit_atm=get_bit_automaton(atm=org_atm, original_bit_width=3)
#bit_atm.draw_graph('bitwise.svg')
print bit_atm.get_summary(logo='bitwise')
strided_b_atm=get_strided_automata2(atm=bit_atm, stride_value=8, is_scalar=True, base_value=2, add_residual=False)
print strided_b_atm.get_summary(logo='strided bitwise')
#strided_b_atm.draw_graph('strided.svg')
strided_b_atm.make_homogenous()
print strided_b_atm.get_summary(logo='homogeneous')
#strided_b_atm.draw_graph('homogeneous.svg', draw_edge_label=True)
minimize_automata(strided_b_atm, same_residuals_only=False)
print strided_b_atm.get_summary(logo='minimized')
strided_b_atm.draw_graph('minimized.svg')
compare_input(True, True, False, None, org_atm, strided_b_atm)
exit(0)
for s in automatas:
stride_dict_list = []
for i in range(4):
symbol_dict, symbol_dictionary_list = get_equivalent_symbols([s], replace=True)
if i == 0:
initial_dic = symbol_dict
my_Automata.add_edge(ste3, ste4)
my_Automata.add_edge(ste4, ste5)
my_Automata.add_edge(ste5, ste6)
my_Automata.add_edge(ste6, ste7)
my_Automata.add_edge(ste7, ste8)
my_Automata.add_edge(ste4, ste9)
my_Automata.add_edge(ste9, ste10)
my_Automata.add_edge(ste10, ste11)
my_Automata.add_edge(ste11, ste8)
my_Automata.draw_graph('t.svg')
my_Automata = my_Automata.get_single_stride_graph()
my_Automata.make_homogenous(plus_src=True)
my_Automata.draw_graph('t2.svg')
my_Automata = my_Automata.get_single_stride_graph()
my_Automata.make_homogenous()
minimize_automata(my_Automata, combine_equal_syms_only=True)
my_Automata.fix_split_all()
my_Automata.draw_graph('s2.svg')
print my_Automata.get_summary()
for n in my_Automata.nodes:
if n.start_type == StartType.fake_root:
continue
s = n.is_symbolset_splitable()
if not s:
print n.id
original_bit_width=8)
#one_bit_atm.make_homogenous()
#minimize_automata(one_bit_atm)
print "finished bit level"
#one_bit_atm.draw_graph("one_bit.svg")
if two_bit:
two_bit_atm = atma.automata_network.get_strided_automata2(
atm=one_bit_atm,
stride_value=2,
is_scalar=True,
base_value=2,
add_residual=False)
two_bit_atm.make_homogenous()
minimize_automata(two_bit_atm)
#two_bit_atm.draw_graph("two_bit.svg")
if three_bit:
three_bit_atm = atma.automata_network.get_strided_automata2(
atm=one_bit_atm,
stride_value=3,
is_scalar=True,
base_value=2,
add_residual=False)
three_bit_atm.make_homogenous()
minimize_automata(three_bit_atm)
#three_bit_atm.draw_graph("three_bit.svg")
if four_bit:
four_bit_atm = atma.automata_network.get_strided_automata2(
from automata.utility.utility import minimize_automata
# Get the usage string
def usage():
usage = "----------------- Usage ----------------\n"
usage += "./count_explicit_states.py <ANML File>"
return usage
# Entry point
if __name__ == '__main__':
# Check the correct number of command line arguments
if len(sys.argv) != 2:
print(usage())
exit(-1)
# Grab the input filename
anml_input = sys.argv[1]
# Parse the ANML file
automata = atma.parse_anml_file(anml_input)
before = automata.nodes_count
# Minimizing the automata with NFA heuristics
minimize_automata(automata)
after = automata.nodes_count
print(str(before) + ", " + str(after))
def process_single_ds(uat):
#uat = AnmalZoo.Ranges05
return_result = {}
result_dir = out_dir + str(uat)
shutil.rmtree(result_dir, ignore_errors=True)
os.mkdir(result_dir)
exempts = {(AnmalZoo.Snort, 1411)}
max_target_stride = 2
uat_count = 200
automatas = atma.parse_anml_file(anml_path[uat])
automatas.remove_ors()
automatas = automatas.get_connected_components_as_automatas()
#uat_count = len(automatas) # comment this to test a subset of automatons defined in uat_count
automatas = automatas[:uat_count]
uat_count = len(automatas)
filed_names = ['number_of_states', 'number_of_edges', 'max_fan_in', 'max_fan_out',
'max_symbol_len', 'min_symbol_len', 'total_sym']
for hom_between, is_Bram in [(False, False), (False, True)]:
for stride_val in range(max_target_stride + 1):
n_states = 0.0
n_edges = 0.0
max_fan_in = 0.0
max_fan_out = 0.0
max_sym_len = 0.0
min_sym_len = 0.0
total_sym = 0.0
with open(result_dir + '/S' + str(stride_val) + '_' + str(uat_count) +
'is_HNH' + str(hom_between) + 'is_Bram' + str(is_Bram) + 'len' + str(len(automatas)) +'.csv', 'w') as csv_file:
csv_writer = csv.writer(csv_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
csv_writer.writerow(filed_names)
for atm_idx, atm in enumerate(automatas):
if (uat, atm_idx) in exempts:
continue
print 'processing {0} stride {3} number {1} from {2}'.format(uat, atm_idx, uat_count, stride_val)
for _ in range(stride_val):
if is_Bram is True and hom_between is True and atm.is_homogeneous is False:
atm.make_homogenous()
atm.make_parentbased_homogeneous()
atm = atm.get_single_stride_graph()
if atm.is_homogeneous is False:
atm.make_homogenous()
minimize_automata(atm)
if is_Bram is True and hom_between is False:
atm.fix_split_all()
all_nodes = filter(lambda n:n.id != 0, atm.nodes) # filter fake root
all_nodes_symbols_len_count = [len(n.symbols) for n in all_nodes]
n_s = atm.nodes_count
n_states += n_s
n_e = atm.edges_count
n_edges += n_e
m_f_i = atm.max_STE_in_degree()
max_fan_in += m_f_i
m_f_o = atm.max_STE_out_degree()
max_fan_out += m_f_o
mx_s_l = max(all_nodes_symbols_len_count)
max_sym_len += mx_s_l
mn_s_l = min(all_nodes_symbols_len_count)
min_sym_len += mn_s_l
t_s = sum(all_nodes_symbols_len_count)
total_sym += t_s
csv_writer.writerow([n_s, n_e, m_f_i, m_f_o, mx_s_l, mn_s_l, t_s])
n_states /= uat_count
n_edges /= uat_count
max_fan_in /= uat_count
max_fan_out /= uat_count
max_sym_len /= uat_count
min_sym_len /= uat_count
total_sym /= uat_count
return_result[(is_Bram, stride_val)] = (n_states,
n_edges,
max_fan_in,
max_fan_out,
max_sym_len,
min_sym_len,
total_sym)
with open(out_dir + 'summary.txt', 'a+') as f:
fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
to_w_lns =[]
to_w_lns.append(str(uat) + "L" + str(uat_count) + "S" + str(stride_val) + "BRam" + str(is_Bram) + "\n")
to_w_lns.append(" average number of states = " + str(n_states) + "\n")
to_w_lns.append(" average number of edges = " + str(n_edges) + "\n")
to_w_lns.append(" average max fan-in = " + str(max_fan_in) + "\n")
to_w_lns.append(" average max fan-out = " + str(max_fan_out) + "\n")
to_w_lns.append(" average max sym-len = " + str(max_sym_len) + "\n")
to_w_lns.append(" average min sym-len = " + str(min_sym_len) + "\n")
to_w_lns.append(" average total_sym-len = " + str(total_sym) + "\n")
f.writelines(to_w_lns)
fcntl.flock(f, fcntl.LOCK_UN)
return uat, return_result
def process_single_ds(uat):
try:
return_result = {}
result_dir = out_dir_prefix + str(uat)
shutil.rmtree(result_dir, ignore_errors=True)
os.mkdir(result_dir)
# cleaning the result folder
automata_per_stage = 50
# this is a pipelineing parameter for staging as pipeline. We usually use 50 for this parameter
automatas = atma.parse_anml_file(anml_path[uat])
automatas.remove_ors()
automatas = automatas.get_connected_components_as_automatas()
uat_count = 100 # number of automata to be processed
# uat_count = len(automatas) # comment this to test a subset of automatons defined in uat_count
automatas = automatas[:uat_count]
uat_count = len(automatas)
number_of_stages = math.ceil(
len(automatas) / float(automata_per_stage))
# number of pipleine stages
atms_per_stage = int(
math.ceil(len(automatas) / float(number_of_stages)))
hdl_folder_name = hd_gen.get_hdl_folder_name(
prefix=str(uat),
number_of_atms=len(automatas),
stride_value=0,
before_match_reg=False,
after_match_reg=False,
ste_type=1,
use_bram=False,
use_compression=False,
compression_depth=-1)
print "folder name to store the HDLs:", hdl_folder_name
generator_ins = hd_gen.HDL_Gen(path=os.path.join(
result_dir, hdl_folder_name),
before_match_reg=False,
after_match_reg=False,
ste_type=1,
total_input_len=dbw)
for atm_idx, atm in enumerate(automatas):
print 'processing {0} number {1} from {2}'.format(
uat, atm_idx + 1, uat_count)
b_atm = atma.automata_network.get_bit_automaton(
atm, original_bit_width=atm.max_val_dim_bits_len)
atm = atma.automata_network.get_strided_automata2(
atm=b_atm,
stride_value=dbw,
is_scalar=True,
base_value=2,
add_residual=True)
atm.make_homogenous()
minimize_automata(atm)
generator_ins.register_automata(atm=atm, use_compression=False)
if (atm_idx + 1) % atms_per_stage == 0:
generator_ins.register_stage_pending(use_bram=False)
generator_ins.finilize()
shutil.make_archive(hdl_folder_name, 'zip', result_dir)
shutil.rmtree(result_dir)
return uat, return_result
except Exception as ex:
print ex
raise ex
if len(automatas) > number_of_autoamtas:
#automatas = random.sample(automatas, number_of_autoamtas)
automatas = automatas[:number_of_autoamtas]
for bit_stride_val in [8]:
strided_automatas, bit_size, = [], []
for atm_idx, atm in enumerate(automatas):
if (uat, atm_idx) in exempts:
continue
old_count = atm.nodes_count
atm = atma.automata_network.get_bit_automaton(
atm=atm, original_bit_width=atm.max_val_dim.bit_length())
#print "finished bitwise"
bit_stride_atm = atma.automata_network.get_strided_automata2(
atm=atm,
stride_value=bit_stride_val,
is_scalar=True,
base_value=2)
#print 'finished {0} stride{3} automata {1} from {2}'.format(uat, atm_idx, len(automatas), bit_stride_val)
if bit_stride_atm.is_homogeneous is False:
bit_stride_atm.make_homogenous(use_espresso=True)
minimize_automata(bit_stride_atm)
print "old bit counts {} new bit count {}".format(
old_count, bit_stride_atm.nodes_count)
