def process_single_ds(uat):
with open(str(uat) + '.txt', "w+") as f:
pass
all_automata = atma.parse_anml_file(anml_path[uat])
all_automata.remove_ors()
automatas = all_automata.get_connected_components_as_automatas()
number_of_autoamtas = len(automatas)
if len(automatas) > number_of_autoamtas:
#automatas = random.sample(automatas, number_of_autoamtas)
automatas = automatas[:number_of_autoamtas]
number_of_autoamtas = len(automatas)
total_bit_len = []
total_bit_hist = {}
for atm_idx, atm in enumerate(automatas):
print 'processing {0} automata {1} from {2}'.format(
uat, atm_idx + 1, number_of_autoamtas)
bc_sym_dict = get_equivalent_symbols([atm], replace=True)
bc_bits_len = int(math.ceil(math.log(max(bc_sym_dict.values()), 2)))
total_bit_len.append(bc_bits_len)
total_bit_hist[bc_bits_len] = total_bit_hist.get(bc_bits_len, 0) + 1
#with open(str(uat) + '.txt', "a") as f:
# print >> f, "compressed bit length: " + str(bc_bits_len)
with open('compression_ratio_summary.txt', "a") as f:
fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
print >> f, "----------------------------------------------------------------"
print >> f, str(uat)
print >> f, "AVG:" + str(
sum(total_bit_len) / float(number_of_autoamtas))
print >> f, "STD:" + str(np.std(total_bit_len))
print >> f, "---------Histogram---------"
for k, v in total_bit_hist.iteritems():
print >> f, "bit len {0} = {1}".format(k, v)
print >> f, "----------------------------------------------------------------"
fcntl.flock(f, fcntl.LOCK_UN)
strided_automatas, bit_size, = [], []
for atm_idx, atm in enumerate(automatas):
if (uat, atm_idx) in exempts:
continue
print atm.get_summary()
atm = atma.automata_network.get_bit_automaton(atm=atm, original_bit_width=hd_gen.HDL_Gen.get_bit_len(atm.max_val_dim))
atm = atma.automata_network.get_strided_automata2(atm=atm, stride_value=bit_stride_val, is_scalar=True,
base_value=2, add_residual=False)
with open('nine_bit_snort.pkl', 'wb') as f:
pickle.dump(atm, f, pickle.HIGHEST_PROTOCOL)
if use_compression:
bc_sym_dict = get_equivalent_symbols([atm], replace=True)
translation_list = []
print atm.get_summary(logo="9bit atm info")
atm = atm.get_single_stride_graph()
print atm.get_summary(logo="done st1")
atm = atm.get_single_stride_graph()
print atm.get_summary(logo="done st2")
atm = atm.get_single_stride_graph()
print atm.get_summary(logo="done st3")
if atm.is_homogeneous is False:
atm.make_homogenous()
minimize_automata(atm)
def process_single_ds(uat):
all_automata = atma.parse_anml_file(anml_path[uat])
all_automata.remove_ors()
automatas = all_automata.get_connected_components_as_automatas()
if len(automatas) > number_of_autoamtas:
#automatas = random.sample(automatas, number_of_autoamtas)
automatas = automatas[:number_of_autoamtas]
number_of_stages = math.ceil(len(automatas) / float(automata_per_stage))
atms_per_stage = int(math.ceil(len(automatas) / float(number_of_stages)))
for stride_val in range(1):
hdl_apth = hd_gen.get_hdl_folder_name(
prefix="comptest" + str(uat),
number_of_atms=len(automatas),
stride_value=stride_val,
before_match_reg=before_match_reg,
after_match_reg=after_match_reg,
ste_type=ste_type,
use_bram=use_bram,
use_compression=use_compression,
compression_depth=compression_depth)
generator_ins = hd_gen.HDL_Gen(
path=hdl_apth,
before_match_reg=before_match_reg,
after_match_reg=after_match_reg,
ste_type=ste_type,
total_input_len=hd_gen.HDL_Gen.get_bit_len(
all_automata.max_val_dim) * pow(2, stride_val))
for atm_idx, atm in enumerate(automatas):
if (uat, atm_idx) in exempts:
continue
print 'processing {0} stride{3} automata {1} from {2}'.format(
uat, atm_idx + 1, len(automatas), stride_val)
bc_bits_len = 8
if use_compression:
bc_sym_dict = get_equivalent_symbols([atm], replace=True)
bc_bits_len = int(
math.ceil(math.log(max(bc_sym_dict.values()), 2)))
translation_list = []
for s in range(stride_val):
atm = atm.get_single_stride_graph()
if use_compression and s < compression_depth:
new_translation = get_equivalent_symbols([atm],
replace=True)
translation_list.append(new_translation)
if atm.is_homogeneous is False:
atm.make_homogenous()
minimize_automata(atm)
#lut_bram_dic = {n: (1, 2) for n in atm.nodes}
generator_ins.register_automata(
atm=atm,
use_compression=use_compression,
byte_trans_map=bc_sym_dict if use_compression else None)
if use_compression:
generator_ins.register_compressor(
[atm.id],
byte_trans_map=bc_sym_dict,
translation_list=translation_list)
if (atm_idx + 1) % atms_per_stage == 0:
generator_ins.register_stage_pending(use_bram=use_bram)
generator_ins.finilize()
#strided_b_atm.draw_graph('homogeneous.svg', draw_edge_label=True)
minimize_automata(strided_b_atm, same_residuals_only=False)
print strided_b_atm.get_summary(logo='minimized')
strided_b_atm.draw_graph('minimized.svg')
compare_input(True, True, False, None, org_atm, strided_b_atm)
exit(0)
for s in automatas:
stride_dict_list = []
for i in range(4):
symbol_dict, symbol_dictionary_list = get_equivalent_symbols([s], replace=True)
if i == 0:
initial_dic = symbol_dict
initial_bits = int(math.ceil(math.log(max(initial_dic.values()), 2)))
width_list=[initial_bits]
test_compressor(original_width=8,
byte_trans_map=initial_dic,
byte_map_width=initial_bits,
translation_list=[],
idx=0,
width_list=[],
initial_width=initial_bits,
output_width=initial_bits)
else:
before_match_reg=before_match_reg,
after_match_reg=after_match_reg,
ste_type=ste_type,
total_input_len=hd_gen.HDL_Gen.get_bit_len(
all_automata.max_val_dim) * pow(2, stride_val))
for atm_idx, atm in enumerate(automatas):
if (uat, atm_idx) in exempts:
continue
print 'processing {0} stride{3} automata {1} from {2}'.format(
uat, atm_idx, len(automatas), stride_val)
bc_bits_len = 8
if use_compression:
bc_sym_dict = get_equivalent_symbols([atm], replace=True)
bc_bits_len = int(
math.ceil(math.log(max(bc_sym_dict.values()), 2)))
translation_list = []
for s in range(stride_val):
atm = atm.get_single_stride_graph()
if use_compression and s < compression_depth:
new_translation = get_equivalent_symbols([atm],
replace=True)
translation_list.append(new_translation)
if atm.is_homogeneous is False:
atm.make_homogenous()