def process_single_ds(uat):
try:
automatas = atma.parse_anml_file(anml_path[uat])
automatas.remove_ors()
if per_cc:
automatas = automatas.get_connected_components_as_automatas()
else:
automatas = [automatas]
report_count, total_states = 0, 0
for atm in automatas:
report_count += atm.number_of_report_nodes
total_states += atm.nodes_count
with open('report_count_summary.txt', "a") as f:
fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
print >> f, "----------------------------------------------------------------"
print >> f, str(uat)
print >> f, "total nodes: " + str(total_states)
print >> f, "total report nodes: " + str(report_count)
print >> f, "----------------------------------------------------------------"
fcntl.flock(f, fcntl.LOCK_UN)
except Exception as e:
print e
def process(inpfile, name):
# Output is: HDL file with the important automata
# selected + table programming.
if inpfile.endswith('mnrl'):
print("ERROR: Entered a MNRL file, likely going to fail")
automatas = atma.parse_anml_file(inpfile)
automata_components = automatas.get_connected_components_as_automatas()
gc.compile_statistics(automata_components)
def process_single_ds(uat):
try:
uat_count = 200
# number of total automata to be processed. Make this number smaller to get results faster for a subset
automatas = atma.parse_anml_file(anml_path[uat])
automatas.remove_ors()
automatas = automatas.get_connected_components_as_automatas()
automatas = automatas[:uat_count]
# pick the number of automataon to be processed. comment this line if you want to process the whole benchmark
target_bit_widths = [1, 2, 4, 16]
# bitwidth to be calculated
stats = [[0, 0] for _ in range(len(target_bit_widths))]
for atm in automatas:
b_atm = atma.automata_network.get_bit_automaton(atm, original_bit_width=atm.max_val_dim_bits_len)
# generate bit automaton
for tb_idx, tb in enumerate(target_bit_widths):
if tb == 1:
atm = b_atm.clone()
else:
atm = atma.automata_network.get_strided_automata2(atm=b_atm,
stride_value=tb,
is_scalar=True,
base_value=2,
add_residual=True)
if atm.is_homogeneous is False:
atm.make_homogenous()
minimize_automata(atm)
n_s = atm.nodes_count
n_e = atm.edges_count
stats[tb_idx][0] += n_s
stats[tb_idx][1] += n_e
global_lock.acquire()
print uat
for tb_idx, tb in enumerate(target_bit_widths):
print "bitwidth = ", tb, "number of states = ", stats[tb_idx][0], "number of edges = ", stats[tb_idx][1]
global_lock.release()
return uat, stats
except Exception, e:
tracebackString = traceback.format_exc(e)
print tracebackString
raise StandardError, "\n\nError occurred. Original traceback is\n%s\n" %(tracebackString)
def process_single_ds(uat):
automatas = atma.parse_anml_file(anml_path[uat])
real_final = [] # these lists keep number of reports for each CC
automatas.remove_ors()
if per_cc:
automatas = automatas.get_connected_components_as_automatas()
else:
automatas = [automatas]
with open(str(uat) + '.txt', "w+") as f:
pass
for atm in automatas:
atm.set_all_symbols_mutation(False)
run_result = automata_run_stat(atm=atm,
file_path=input_path10M[uat],
cycle_detail=False,
report_detail=True,
bytes_per_dim=1,
iterations=iterations)
real_final.append(run_result[total_reports])
with open(str(uat) + '.txt', "a") as f:
print >> f, "total reports: " + str(real_final[-1])
print >> f, "real nodes count: " + str(atm.nodes_count)
print >> f, "reports per cycle: " + str(
run_result[reports_per_cycle])
print >> f, "reports in cycle: " + str(
run_result[reports_in_cycle])
print >> f, "----------------------------------------------------------------"
with open('summary.txt', "a") as f:
fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
print >> f, "----------------------------------------------------------------"
print >> f, str(uat)
print >> f, "total cycles: " + str(iterations)
print >> f, "total number of reports: " + str(sum(real_final))
print >> f, "total number of report cycles: " + str(
sum(run_result[reports_in_cycle]))
print >> f, "average number of reports per cycle: " + str(
sum(real_final) / float(iterations))
print >> f, "average number of report cycle: " + str(
sum(run_result[reports_in_cycle]) / float(iterations))
print >> f, "----------------------------------------------------------------"
fcntl.flock(f, fcntl.LOCK_UN)
def process_single_ds(uat):
automata_per_stage = 50
exempts = {(AnmalZoo.Snort, 1411)}
automatas = atma.parse_anml_file(anml_path[uat])
automatas.remove_ors()
automatas = automatas.get_connected_components_as_automatas()
uat_count = len(automatas)
automatas = automatas[:uat_count]
uat_count = len(automatas)
number_of_stages = math.ceil(len(automatas) / float(automata_per_stage))
atms_per_stage = int(math.ceil(len(automatas) / float(number_of_stages)))
hdl_folder_name = hd_gen.get_hdl_folder_name(prefix=str(uat), number_of_atms=uat_count,
stride_value=stride, before_match_reg=False,
after_match_reg=False, ste_type=1, use_bram=False,
use_compression=False, compression_depth=-1,
use_mid_fifo=False, use_rst=True)
generator_ins = hd_gen.HDL_Gen(path=os.path.join("/home/gr5yf/FCCM_2020/lut16", hdl_folder_name), before_match_reg=False,
after_match_reg=False, ste_type=1,
total_input_len=automatas[0].max_val_dim_bits_len * pow(2, stride),
bram_shape=None)
for atm_idx, atm in enumerate(automatas):
if (uat, atm_idx) in exempts:
continue
print 'processing {0} stride {3} number {1} from {2}'.format(uat, atm_idx, uat_count, stride)
atm = atm.get_single_stride_graph()
atm = atm.get_single_stride_graph()
atm.make_homogenous()
minimize_automata(atm)
generator_ins.register_automata(atm=atm, use_compression=False, lut_bram_dic={})
if (atm_idx + 1) % atms_per_stage == 0:
generator_ins.register_stage_pending(use_bram=False)
generator_ins.register_stage_pending(use_bram=False)
generator_ins.finilize(dataplane_intcon_max_degree=5, contplane_intcon_max_degree=10)
def extract_automata_components(file_groups, opts):
automata_components = []
index = 0
for file in file_groups:
if not file.endswith('.anml'):
continue
if opts.print_file_info:
print "Group Index: ", index, "comes from file", file
index += 1
automatas = atma.parse_anml_file(file)
# Need to remove ors to avoid 'special elements' that
# prevent us from getting the connected components.
automatas.remove_ors()
automata_components.append(
automatas.get_connected_components_as_automatas())
return automata_components
def process_single_ds(uat):
with open(str(uat) + '.txt', "w+") as f:
pass
all_automata = atma.parse_anml_file(anml_path[uat])
all_automata.remove_ors()
automatas = all_automata.get_connected_components_as_automatas()
number_of_autoamtas = len(automatas)
if len(automatas) > number_of_autoamtas:
#automatas = random.sample(automatas, number_of_autoamtas)
automatas = automatas[:number_of_autoamtas]
number_of_autoamtas = len(automatas)
total_bit_len = []
total_bit_hist = {}
for atm_idx, atm in enumerate(automatas):
print 'processing {0} automata {1} from {2}'.format(
uat, atm_idx + 1, number_of_autoamtas)
bc_sym_dict = get_equivalent_symbols([atm], replace=True)
bc_bits_len = int(math.ceil(math.log(max(bc_sym_dict.values()), 2)))
total_bit_len.append(bc_bits_len)
total_bit_hist[bc_bits_len] = total_bit_hist.get(bc_bits_len, 0) + 1
#with open(str(uat) + '.txt', "a") as f:
# print >> f, "compressed bit length: " + str(bc_bits_len)
with open('compression_ratio_summary.txt', "a") as f:
fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
print >> f, "----------------------------------------------------------------"
print >> f, str(uat)
print >> f, "AVG:" + str(
sum(total_bit_len) / float(number_of_autoamtas))
print >> f, "STD:" + str(np.std(total_bit_len))
print >> f, "---------Histogram---------"
for k, v in total_bit_hist.iteritems():
print >> f, "bit len {0} = {1}".format(k, v)
print >> f, "----------------------------------------------------------------"
fcntl.flock(f, fcntl.LOCK_UN)
def process_single_ds(uat):
automatas = atma.parse_anml_file(anml_path[uat])
approximate_final, real_final = [], [] # these lists keep number of reports for each CC
real_states, appr_states = 0, 0 # these integers count number of states
translation_dic = {x: x % approximate_ratio for x in range(automatas.max_val_dim + 1)}
automatas.remove_ors()
automatas = automatas.get_connected_components_as_automatas()
with open(str(uat) + '.txt', "w+") as f:
pass
for atm in automatas[:uat_count]:
real_states+= atm.nodes_count
atm.set_all_symbols_mutation(False)
appr_automata = get_approximate_automata(atm=atm, translation_dic=translation_dic,
max_val_dim=atm.max_val_dim / approximate_ratio)
minimize_automata(automata=appr_automata)
appr_states += appr_automata.nodes_count
run_result = automata_run_stat(atm=atm, file_path=input_path[uat], cycle_detail=True, report_detail=False, bytes_per_dim=1)
real_final.append(run_result[total_reports])
appr_run_result = automata_run_stat(atm=appr_automata, file_path=input_path[uat], cycle_detail=True, report_detail=False, bytes_per_dim=1,
translation_dic=translation_dic)
approximate_final.append(appr_run_result[total_reports])
with open(str(uat) + '.txt', "a") as f:
print >> f, "real reports: " + str(real_final[-1])
print >> f, "approximate reports: " + str(approximate_final[-1])
print >> f, "real nodes count: " + str(atm.nodes_count)
print >> f, "approximate nodes count:" + str(appr_automata.nodes_count)
print >>f, "----------------------------------------------------------------"
with open(str(uat) + '.ttxt', "a") as f:
print >>f, "***************sum*******************"
print >>f, "real reports: " + str(sum(real_final))
print >>f, "approximate reports: " + str(sum(approximate_final))
print >>f, "real nodes count: " + str(real_states)
print >>f, "approximate nodes count:" + str(appr_states)
def process_single_ds(uat):
automatas = atma.parse_anml_file(anml_path[uat])
automatas.remove_ors()
automatas = automatas.get_connected_components_as_automatas()
uat_count = len(automatas)
n_states = 0.0
n_edges = 0.0
total_sym = 0.0
for atm_idx, atm in enumerate(automatas):
minimize_automata(atm)
all_nodes = filter(lambda n: n.id != 0, atm.nodes) # filter fake root
all_nodes_symbols_len_count = [
len(list(n.symbols.points)) for n in all_nodes
]
n_s = atm.nodes_count
n_states += n_s
n_e = atm.edges_count
n_edges += n_e
t_s = sum(all_nodes_symbols_len_count)
total_sym += t_s
global_lock.acquire()
print str(uat), "\t nodes count = ", n_states, "\tedges count = ", n_edges, "\tAvg node degree = ",\
2 * n_edges / n_states, "\tAvg symbol size = ", total_sym/n_states
global_lock.release()
return True
#SynthBring, Synthcorering
under_process_atms = [AnmalZoo.EntityResolution]
exempts = {(AnmalZoo.Snort, 1411)}
number_of_autoamtas = 200
automata_per_stage = 50
use_compression = False
single_out=False
before_match_reg=False
after_match_reg=False
ste_type=1
use_bram=False
compression_depth = 0
for uat in under_process_atms:
automatas = atma.parse_anml_file(anml_path[uat])
automatas.remove_ors()
automatas = automatas.get_connected_components_as_automatas()
#automatas=pickle.load(open('Snort1-10.pkl', 'rb'))
if len(automatas) > number_of_autoamtas:
#automatas = random.sample(automatas, number_of_autoamtas)
automatas = automatas[:number_of_autoamtas]
number_of_stages = math.ceil(len(automatas) / float(automata_per_stage))
atms_per_stage = int(math.ceil(len(automatas) / float(number_of_stages)))
for bit_stride_val in [9]:
hdl_apth = hd_gen.get_hdl_folder_name(prefix=str(uat), number_of_atms=len(automatas), stride_value=bit_stride_val,
def process_single_ds(uat):
#uat = AnmalZoo.Ranges05
return_result = {}
result_dir = out_dir + str(uat)
shutil.rmtree(result_dir, ignore_errors=True)
os.mkdir(result_dir)
exempts = {(AnmalZoo.Snort, 1411)}
max_target_stride = 2
uat_count = 200
automata_per_stage = 50
automatas = atma.parse_anml_file(anml_path[uat])
automatas.remove_ors()
automatas = automatas.get_connected_components_as_automatas()
after_match_reg = False
actual_bram = False # if True, actual bram will be used. Otherwise, LUT emulates bram
#uat_count = len(automatas) # comment this to test a subset of automatons defined in uat_count
automatas = automatas[:uat_count]
uat_count = len(automatas)
number_of_stages = math.ceil(len(automatas) / float(automata_per_stage))
atms_per_stage = int(math.ceil(len(automatas) / float(number_of_stages)))
for hom_between, is_Bram in [(False, True)]:
hdl_writers = []
for i in range(max_target_stride + 1):
hdl_folder_name = hd_gen.get_hdl_folder_name(prefix=str(uat), number_of_atms=len(automatas),
stride_value=i, before_match_reg=False,
after_match_reg=after_match_reg, ste_type=1, use_bram=is_Bram,
use_compression=False, compression_depth=-1)
generator_ins = hd_gen.HDL_Gen(path=os.path.join(result_dir, hdl_folder_name), before_match_reg=False,
after_match_reg=after_match_reg, ste_type=1,
total_input_len=4 * pow(2, i),
bram_shape=(512, 36))
hdl_writers.append(generator_ins)
for atm_idx, atm in enumerate(automatas):
b_atm = atma.automata_network.get_bit_automaton(atm, original_bit_width=atm.max_val_dim_bits_len)
atm = atma.automata_network.get_strided_automata2(atm=b_atm,
stride_value=4,
is_scalar=True,
base_value=2,
add_residual=True)
for stride_val in reversed(range(max_target_stride + 1)):
if (uat, atm_idx) in exempts:
continue
print 'processing {0} stride {3} number {1} from {2}'.format(uat, atm_idx, uat_count, stride_val)
s_atm = atm
for _ in range(stride_val):
if s_atm is atm:
s_atm = atm.get_single_stride_graph()
else:
s_atm = s_atm.get_single_stride_graph()
if s_atm.is_homogeneous is False:
s_atm.make_homogenous()
minimize_automata(s_atm)
if is_Bram is True and hom_between is False:
s_atm.fix_split_all()
if is_Bram:
lut_bram_dic = {n: tuple((2 for _ in range(s_atm.stride_value))) for n in s_atm.nodes if
n.is_fake is False}
else:
lut_bram_dic = {}
hdl_writers[stride_val].register_automata(atm=s_atm, use_compression=False, lut_bram_dic=lut_bram_dic)
if (atm_idx + 1) % atms_per_stage == 0:
hdl_writers[stride_val].register_stage_pending(single_out=False, use_bram=actual_bram)
for i in range(max_target_stride + 1):
hdl_writers[i].finilize()
return uat, return_result
def process_single_ds(uat):
return_result = {}
#result_dir = out_dir_prefix + str(uat)
#shutil.rmtree(result_dir, ignore_errors=True)
#os.mkdir(result_dir)
# cleaning the result folder
automata_per_stage = 100
# this is a pipelineing parameter for staging as pipeline. We usually use 50 for this parameter
automatas = atma.parse_anml_file(anml_path[uat])
automatas.remove_ors()
automatas = automatas.get_connected_components_as_automatas()
#uat_count = 10 # number of automata to be processed
uat_count = len(
automatas
) # comment this to test a subset of automatons defined in uat_count
automatas = automatas[:uat_count]
uat_count = len(automatas)
number_of_stages = int(
math.ceil(len(automatas) / float(automata_per_stage)))
# number of pipleine stages
atms_per_stage = int(math.ceil(len(automatas) / float(number_of_stages)))
hdl_folder_name = hd_gen.get_hdl_folder_name(prefix=str(uat),
number_of_atms=len(automatas),
stride_value=0,
before_match_reg=False,
after_match_reg=False,
ste_type=1,
use_bram=True,
use_compression=False,
compression_depth=-1,
use_mid_fifo=use_mid_fifo,
use_rst=use_rst)
print "folder name to store the HDLs:", hdl_folder_name
generator_ins = hd_gen.HDL_Gen(path=os.path.join(
"/home/gr5yf/FCCM_2020/bram8", hdl_folder_name),
before_match_reg=False,
after_match_reg=False,
ste_type=1,
total_input_len=dbw,
use_mid_fifo=use_mid_fifo,
use_rst=use_rst,
bram_shape=(512, 36))
for atm_idx, atm in enumerate(automatas):
print 'processing {0} number {1} from {2}'.format(
uat, atm_idx + 1, uat_count)
minimize_automata(atm)
lut_bram_dic = {
n: tuple((2 for _ in range(atm.stride_value)))
for n in atm.nodes if n.is_fake is False
}
generator_ins.register_automata(atm=atm,
use_compression=False,
lut_bram_dic=lut_bram_dic)
if (atm_idx + 1) % atms_per_stage == 0:
generator_ins.register_stage_pending(use_bram=True)
generator_ins.register_stage_pending(use_bram=True)
generator_ins.finilize(dataplane_intcon_max_degree=5,
contplane_intcon_max_degree=10)
return uat, return_result
def process_anml(bitwidth, input_directory, automata_per_stage):
# This is the directory name to be created for HDL files
output_hdl_directory = input_directory + '/' + str(bitwidth) + '_' + str(
automata_per_stage)
anml_input_files = glob.glob(input_directory + '/*.anml')
# Clean up directory
shutil.rmtree(output_hdl_directory, ignore_errors=True)
os.mkdir(output_hdl_directory)
# Create a directory name for the HDL code
hdl_folder_name = hd_gen.get_hdl_folder_name(
prefix=output_hdl_directory,
number_of_atms=len(anml_input_files),
stride_value=0,
before_match_reg=False,
after_match_reg=False,
ste_type=1,
use_bram=False,
use_compression=False,
compression_depth=-1,
symbolic=False)
# Create a hardware Generator
generator_ins = hd_gen.HDL_Gen(path=hdl_folder_name,
before_match_reg=False,
after_match_reg=False,
ste_type=1,
total_input_len=dbw,
symbolic=False)
# Iterate through the ANML files in the directory
for index, anml_input_file in enumerate(anml_input_files):
# Grab the automata file number
automata_number = re.search('\d+', anml_input_file).group(0)
# Parse the ANML file
automata = atma.parse_anml_file(anml_input_file)
if dbw == 16:
print "Doing 16-bit!"
automata_with_set_bw = automata.get_single_stride_graph()
else:
print "Doing 8-bit!"
automata_with_set_bw = automata
assert dbw == automata_with_set_bw.total_bits_len, "Bitwidth assumption is incorrect!"
if not automata_with_set_bw.is_homogeneous:
print "Converting to homogeneous automaton"
automata_with_set_bw.make_homogenous()
# Minimizing the automata with NFA heuristics
if minimize:
minimize_automata(automata_with_set_bw)
#atma.generate_anml_file(anml_input_file + "_min.anml", automata)
else:
print("No minimization of Automata")
# Drawing automata graph
if drawing:
print "Drawing automata svg graph"
automata_with_set_bw.draw_graph(anml_input_file +
"_minimized_hw.svg")
automata_info.append('{},{},{}\n'.format(
automata_number, str(automata_with_set_bw.nodes_count),
str(automata_with_set_bw.edges_count)))
# # Register this automaton
generator_ins.register_automata(atm=automata_with_set_bw,
use_compression=False)
# We've got another batch of automata_per_stage automata to stage
if (index + 1) % automata_per_stage == 0:
generator_ins.register_stage_pending(single_out=False,
use_bram=False)
# DO we need this? maybe if our number of automata is not a perfect multiple
# of automata_per_stage?
generator_ins.register_stage_pending(single_out=False, use_bram=False)
#Finalize and wrap up HDL in archive folder
generator_ins.finilize()
# Using gztar to handle LARGE automata workloads
shutil.make_archive(hdl_folder_name, 'gztar', output_hdl_directory)
shutil.rmtree(output_hdl_directory)
# Write the automata node and edge count to a file
with open(hdl_folder_name + '.stats', 'w') as output_file:
output_file.write("Number of States, Number of Edges\n")
output_file.write("---------------------------------\n")
for automata_string in automata_info:
output_file.write(automata_string)
def process_anml(bitwidth, input_directory, automata_per_stage):
# This is the directory name to be created for HDL files
output_hdl_directory = input_directory + '/' + str(bitwidth) + '_' + str(
automata_per_stage)
anml_input_files = glob.glob(input_directory + '/*.anml')
print("ANML Files: ", anml_input_files)
# Clean up directory
shutil.rmtree(output_hdl_directory, ignore_errors=True)
os.mkdir(output_hdl_directory)
SYMBOLIC = True
# Create a directory name for the HDL code
hdl_folder_name = hd_gen.get_hdl_folder_name(
prefix=output_hdl_directory,
number_of_atms=len(anml_input_files),
stride_value=0,
before_match_reg=False,
after_match_reg=False,
ste_type=1,
use_bram=False,
use_compression=False,
compression_depth=-1,
symbolic=SYMBOLIC)
print("Folder name to store the HDLs: ", hdl_folder_name)
# Create a hardware Generator
generator_ins = hd_gen.HDL_Gen(path=hdl_folder_name,
before_match_reg=False,
after_match_reg=False,
ste_type=1,
total_input_len=dbw,
symbolic=SYMBOLIC)
# Iterate through the ANML files in the directory
for index, anml_input_file in enumerate(anml_input_files):
print("Parsing Automata: {}".format(anml_input_file))
module_name = 'Automata_tt_' + str(index)
verilog_filename = hdl_folder_name + '/automata_tt_' + str(
index) + '.sv'
# Parse the ANML file
automata = atma.parse_anml_file(anml_input_file)
# Minimizing the automata with NFA heuristics
if minimize:
print("Minimizing Automata: {}".format(anml_input_file))
minimize_automata(automata)
#atma.generate_anml_file(anml_input_file + "_min.anml", automata)
else:
print("No minimization of Automata")
tables, start_states, accept_states = atma.generate_tt(automata)
inputs = atma.build_truthtable(tables, start_states, accept_states,
module_name, verilog_filename)
print "Inputs: ", inputs
# for now, we will use this automata proxy
automata = Automatanetwork('tt_' + str(index),
True,
1,
255,
inputs=inputs)
new_node = S_T_E(start_type=StartType.non_start,
is_report=True,
is_marked=False,
id=automata.get_new_id(),
symbol_set=None,
adjacent_S_T_E_s=None,
report_residual=1,
report_code=1)
automata.add_element(new_node)
print "Number of states: ", automata.nodes_count
# Register this automaton
generator_ins.register_automata(atm=automata, use_compression=False)
# We've got another batch of automata_per_stage automata to stage
if (index + 1) % automata_per_stage == 0:
generator_ins.register_stage_pending(single_out=False,
use_bram=False)
# DO we need this? maybe if our number of automata is not a perfect multiple
# of automata_per_stage?
generator_ins.register_stage_pending(single_out=False, use_bram=False)
#Finalize and wrap up HDL in archive folder
generator_ins.finilize()
# Using gztar to handle LARGE automata workloads
shutil.make_archive(hdl_folder_name, 'gztar', output_hdl_directory)
shutil.rmtree(output_hdl_directory)
def process_single_ds(uat):
return_result = {}
result_dir = out_dir + str(uat)
shutil.rmtree(result_dir, ignore_errors=True)
os.mkdir(result_dir)
exempts = {(AnmalZoo.Snort, 1411)}
max_target_stride = 2
uat_count = 200
automata_per_stage = 50
automatas = atma.parse_anml_file(anml_path[uat])
automatas.remove_ors()
automatas = automatas.get_connected_components_as_automatas()
after_match_reg = False
actual_bram = False # if True, actual bram will be used. Otherwise, LUT emulates bram
automatas = automatas[:uat_count]
uat_count = len(automatas)
number_of_stages = math.ceil(len(automatas) / float(automata_per_stage))
atms_per_stage = int(math.ceil(len(automatas) / float(number_of_stages)))
for hom_between, is_Bram in [(False, True)]:
for stride_val in range(max_target_stride + 1):
hdl_folder_name = hd_gen.get_hdl_folder_name(
prefix=str(uat),
number_of_atms=len(automatas),
stride_value=stride_val,
before_match_reg=False,
after_match_reg=after_match_reg,
ste_type=1,
use_bram=is_Bram,
use_compression=False,
compression_depth=-1)
generator_ins = hd_gen.HDL_Gen(
path=os.path.join(result_dir, hdl_folder_name),
before_match_reg=False,
after_match_reg=False,
ste_type=1,
total_input_len=automatas[0].max_val_dim_bits_len *
pow(2, stride_val),
bram_shape=(512, 36))
for atm_idx, atm in enumerate(automatas):
if (uat, atm_idx) in exempts:
continue
print 'processing {0} stride {3} number {1} from {2}'.format(
uat, atm_idx, uat_count, stride_val)
for _ in range(stride_val):
if is_Bram is True and hom_between is True and atm.is_homogeneous is False:
atm.make_homogenous()
atm.make_parentbased_homogeneous()
atm = atm.get_single_stride_graph()
if atm.is_homogeneous is False:
atm.make_homogenous()
minimize_automata(atm)
if is_Bram is True and hom_between is False:
atm.fix_split_all()
if is_Bram:
lut_bram_dic = {
n: tuple((2 for _ in range(atm.stride_value)))
for n in atm.nodes if n.is_fake is False
}
else:
lut_bram_dic = {}
generator_ins.register_automata(atm=atm,
use_compression=False,
lut_bram_dic=lut_bram_dic)
if (atm_idx + 1) % atms_per_stage == 0:
generator_ins.register_stage_pending(single_out=False,
use_bram=actual_bram)
generator_ins.finilize()
return uat, return_result
compression_depth=-1)
#print "Folder name to store the HDLs: ", hdl_folder_name
# Create a hardware Generator
generator_ins = hd_gen.HDL_Gen(path=os.path.join(output_hdl_directory,
hdl_folder_name),
before_match_reg=False,
after_match_reg=False,
ste_type=1,
total_input_len=dbw)
#print "ANML Input File: ", input_anml_file
# Parse the ANML file
automata = atma.parse_anml_file(input_anml_file)
# Iterate through the ANML files in the directory
for index in range(num_automata):
temp_automata = automata.clone()
# Assign each unique automaton its own unique name or the HDL generator won't work
temp_automata.id = 'an{}'.format(index)
generator_ins.register_automata(atm=temp_automata,
use_compression=False)
# Once all of the automata have been registered, add registers
generator_ins.register_stage_pending(single_out=False, use_bram=False)
import automata as atma
automata = atma.parse_anml_file("/home/reza/Git/ANMLZoo/ClamAV/anml/515_nocounter.1chip.anml")
print "Finished processing ClamAV"
automata.remove_ors()
automata.print_summary()
print "connecting component size=", automata.get_connected_components_size()
automata = atma.parse_anml_file("/home/reza/Git/ANMLZoo/Dotstar/anml/backdoor_dotstar.1chip.anml")
print "Finished processing Dotstar"
automata.remove_ors()
automata.print_summary()
print "connecting component size=", automata.get_connected_components_size()
automata = atma.parse_anml_file("/home/reza/Git/ANMLZoo/EntityResolution/anml/1000.1chip.anml")
print "Finished processing Entity Resolution"
automata.remove_ors()
automata.print_summary()
print "connecting component size=", automata.get_connected_components_size()
automata = atma.parse_anml_file("/home/reza/Git/ANMLZoo/Fermi/anml/fermi_2400.1chip.anml")
print "Finished processing fermi"
automata.remove_ors()
automata.print_summary()
print "connecting component size=", automata.get_connected_components_size()
automata = atma.parse_anml_file("/home/reza/Git/ANMLZoo/Hamming/anml/93_20X3.1chip.anml")
def thread_func(ds):
try:
full_atm = atma.parse_anml_file(anml_path[ds])
full_atm.remove_ors()
atms_list = full_atm.get_connected_components_as_automatas()
atms_list = atms_list[:atms_count]
with open(str(ds) + '4bit.csv', 'w') as csv_file:
csv_writer = csv.writer(csv_file,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
csv_writer.writerow(filed_names)
for curr_atm in atms_list:
b_atm = atma.automata_network.get_bit_automaton(
curr_atm, original_bit_width=curr_atm.max_val_dim_bits_len)
four_b_atm = atma.automata_network.get_strided_automata2(
atm=b_atm,
stride_value=4,
is_scalar=True,
base_value=2,
add_residual=True)
curr_atm = four_b_atm.get_single_stride_graph()
curr_atm.make_homogenous()
minimize_automata(curr_atm)
curr_atm.fix_split_all()
match_graph = nx.Graph()
parent_sym_to_product_sym_dic = {}
for ste in curr_atm.nodes:
if ste.is_fake:
continue # fake root is not considered
match_graph.add_node(ste)
# here we find the union of parents and find the version with false posetive versions
comb_sym_set = PackedIntervalSet(
[]
) # create an empty symbol sets to find union of parents symbol set
for pred in curr_atm.get_predecessors(ste):
if pred.is_fake:
continue
for ivl in pred.symbols.intervals:
comb_sym_set.add_interval(
ivl
) # finding the union of parents symbol sets
# making symbol set smaller
comb_sym_set.prone()
comb_sym_set.merge()
comb_sym_set = comb_sym_set.get_combinatorial_symbol_set()
parent_sym_to_product_sym_dic[ste] = comb_sym_set
# now having a graph with no edge, we add edge between nodes if they can be combined with each other
# Two nodes can be combined if these two conditions are satisfied. First, they should not have a common symbol
# second, their parrents should also not have common symbols
for n in match_graph.nodes():
if n == ste:
continue # we do not check a node with itself
if is_there_common_sym(ste.symbols, n.symbols):
continue # first condition has not been met
elif is_there_common_sym(
parent_sym_to_product_sym_dic[ste],
parent_sym_to_product_sym_dic[n]):
continue # second condition has not been met
else: # both condiotions have been met, we can create an edge between ste, n
match_graph.add_edge(ste, n)
matching_result = matching_alg.max_weight_matching(match_graph)
print "number of orignal nodes ", curr_atm.nodes_count
print "number of matching nodes ", 2 * len(matching_result)
csv_writer.writerow(
[curr_atm.nodes_count, 2 * len(matching_result)])
except Exception as ex:
print traceback.print_exc()
def process_single_ds(uat):
exempts = {(AnmalZoo.Snort, 1411)}
automata_per_stage = 50
# this is a pipelineing parameter for staging as pipeline. We usually use 50 for this parameter
automatas = atma.parse_anml_file(anml_path[uat])
automatas.remove_ors()
automatas = automatas.get_connected_components_as_automatas()
#uat_count = 10 # number of automata to be processed
uat_count = len(
automatas
) # comment this to test a subset of automatons defined in uat_count
automatas = automatas[:uat_count]
uat_count = len(automatas)
number_of_stages = int(
math.ceil(len(automatas) / float(automata_per_stage)))
# number of pipleine stages
atms_per_stage = int(math.ceil(len(automatas) / float(number_of_stages)))
hdl_folder_name = hd_gen.get_hdl_folder_name(prefix=str(uat),
number_of_atms=len(automatas),
stride_value=stride,
before_match_reg=False,
after_match_reg=False,
ste_type=1,
use_bram=True,
use_compression=False,
compression_depth=-1,
use_mid_fifo=use_mid_fifo,
use_rst=use_rst)
print "folder name to store the HDLs:", hdl_folder_name
generator_ins = hd_gen.HDL_Gen(
path=os.path.join("/home/gr5yf/FCCM_2020/bram16", hdl_folder_name),
before_match_reg=False,
after_match_reg=False,
ste_type=1,
total_input_len=automatas[0].max_val_dim_bits_len * pow(2, stride),
use_mid_fifo=use_mid_fifo,
use_rst=use_rst,
bram_shape=(512, 36))
for atm_idx, atm in enumerate(automatas):
if (uat, atm_idx) in exempts:
continue
print 'processing {0} number {1} from {2}'.format(
uat, atm_idx + 1, uat_count)
for s in range(stride):
atm = atm.get_single_stride_graph()
atm.make_homogenous()
minimize_automata(atm)
atm.fix_split_all()
lut_bram_dic = {
n: tuple((2 for _ in range(atm.stride_value)))
for n in atm.nodes if n.is_fake is False
}
generator_ins.register_automata(atm=atm,
use_compression=False,
lut_bram_dic=lut_bram_dic)
if (atm_idx + 1) % atms_per_stage == 0:
generator_ins.register_stage_pending(use_bram=True)
generator_ins.register_stage_pending(use_bram=True)
generator_ins.finilize(dataplane_intcon_max_degree=5,
contplane_intcon_max_degree=10)
return
def process_single_ds(uat):
try:
#uat = AnmalZoo.Ranges05
return_result = {}
result_dir = out_dir + str(uat)
shutil.rmtree(result_dir, ignore_errors=True)
os.mkdir(result_dir)
exempts = {(AnmalZoo.Snort, 1411)}
min_target_stride, max_target_stride = 3, 3
uat_count = 200
automatas = atma.parse_anml_file(anml_path[uat])
automatas.remove_ors()
automatas = automatas.get_connected_components_as_automatas()
#uat_count = len(automatas) # comment this to test a subset of automatons defined in uat_count
automatas = automatas[:uat_count]
uat_count = len(automatas)
filed_names = [
'number_of_states', 'number_of_edges', 'max_fan_in', 'max_fan_out',
'max_symbol_len', 'min_symbol_len', 'total_sym'
]
for hom_between, is_Bram in [(False, True)]:
n_states = [0.0 for _ in range(max_target_stride + 1)]
n_edges = [0.0 for _ in range(max_target_stride + 1)]
max_fan_in = [0.0 for _ in range(max_target_stride + 1)]
max_fan_out = [0.0 for _ in range(max_target_stride + 1)]
max_sym_len = [0.0 for _ in range(max_target_stride + 1)]
min_sym_len = [0.0 for _ in range(max_target_stride + 1)]
total_sym = [0.0 for _ in range(max_target_stride + 1)]
csv_writers = []
for i in range(max_target_stride + 1):
f = open(
result_dir + '/S' + str(i) + '_' + str(uat_count) +
'is_HNH' + str(hom_between) + 'is_Bram' + str(is_Bram) +
'len' + str(uat_count) + '.csv', 'w')
csv_writer = csv.writer(f,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
csv_writer.writerow(filed_names)
csv_writers.append(csv_writer)
for atm_idx, atm in enumerate(automatas):
b_atm = atma.automata_network.get_bit_automaton(
atm, original_bit_width=atm.max_val_dim_bits_len)
atm = atma.automata_network.get_strided_automata2(
atm=b_atm,
stride_value=4,
is_scalar=True,
base_value=2,
add_residual=True)
for stride_val in reversed(
range(min_target_stride, max_target_stride + 1)):
if (uat, atm_idx) in exempts:
continue
print 'processing {0} stride {3} number {1} from {2}'.format(
uat, atm_idx, uat_count, stride_val)
s_atm = atm
for _ in range(stride_val):
if s_atm is atm:
s_atm = atm.get_single_stride_graph()
else:
s_atm = s_atm.get_single_stride_graph()
if s_atm.is_homogeneous is False:
s_atm.make_homogenous()
minimize_automata(s_atm)
if is_Bram is True and hom_between is False:
s_atm.fix_split_all()
all_nodes = filter(lambda n: n.id != 0,
s_atm.nodes) # filter fake root
all_nodes_symbols_len_count = [
len(n.symbols) for n in all_nodes
]
n_s = s_atm.nodes_count
n_states[stride_val] += n_s
n_e = s_atm.edges_count
n_edges[stride_val] += n_e
m_f_i = s_atm.max_STE_in_degree()
max_fan_in[stride_val] += m_f_i
m_f_o = s_atm.max_STE_out_degree()
max_fan_out[stride_val] += m_f_o
mx_s_l = max(all_nodes_symbols_len_count)
max_sym_len[stride_val] += mx_s_l
mn_s_l = min(all_nodes_symbols_len_count)
min_sym_len[stride_val] += mn_s_l
t_s = sum(all_nodes_symbols_len_count)
total_sym[stride_val] += t_s
csv_writers[stride_val].writerow(
[n_s, n_e, m_f_i, m_f_o, mx_s_l, mn_s_l, t_s])
del csv_writers
for i in range(max_target_stride + 1):
n_states[i] /= uat_count
n_edges[i] /= uat_count
max_fan_in[i] /= uat_count
max_fan_out[i] /= uat_count
max_sym_len[i] /= uat_count
min_sym_len[i] /= uat_count
total_sym[i] /= uat_count
return_result[(is_Bram, i)] = (n_states, n_edges, max_fan_in,
max_fan_out, max_sym_len,
min_sym_len, total_sym)
with open(out_dir + 'summary.txt', 'a+') as f:
fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
to_w_lns = []
to_w_lns.append(
str(uat) + "L" + str(uat_count) + "S" + str(i) +
"BRam" + str(is_Bram) + "\n")
to_w_lns.append(" average number of states = " +
str(n_states[i]) + "\n")
to_w_lns.append(" average number of edges = " +
str(n_edges[i]) + "\n")
to_w_lns.append(" average max fan-in = " +
str(max_fan_in[i]) + "\n")
to_w_lns.append(" average max fan-out = " +
str(max_fan_out[i]) + "\n")
to_w_lns.append(" average max sym-len = " +
str(max_sym_len[i]) + "\n")
to_w_lns.append(" average min sym-len = " +
str(min_sym_len[i]) + "\n")
to_w_lns.append(" average total_sym-len = " +
str(total_sym[i]) + "\n")
f.writelines(to_w_lns)
fcntl.flock(f, fcntl.LOCK_UN)
return uat, return_result
except Exception, e:
tracebackString = traceback.format_exc(e)
print tracebackString
raise StandardError, "\n\nError occurred. Original traceback is\n%s\n" % (
tracebackString)
def process_single_ds(uat):
all_automata = atma.parse_anml_file(anml_path[uat])
all_automata.remove_ors()
automatas = all_automata.get_connected_components_as_automatas()
if len(automatas) > number_of_autoamtas:
#automatas = random.sample(automatas, number_of_autoamtas)
automatas = automatas[:number_of_autoamtas]
number_of_stages = math.ceil(len(automatas) / float(automata_per_stage))
atms_per_stage = int(math.ceil(len(automatas) / float(number_of_stages)))
for stride_val in range(1):
hdl_apth = hd_gen.get_hdl_folder_name(
prefix="comptest" + str(uat),
number_of_atms=len(automatas),
stride_value=stride_val,
before_match_reg=before_match_reg,
after_match_reg=after_match_reg,
ste_type=ste_type,
use_bram=use_bram,
use_compression=use_compression,
compression_depth=compression_depth)
generator_ins = hd_gen.HDL_Gen(
path=hdl_apth,
before_match_reg=before_match_reg,
after_match_reg=after_match_reg,
ste_type=ste_type,
total_input_len=hd_gen.HDL_Gen.get_bit_len(
all_automata.max_val_dim) * pow(2, stride_val))
for atm_idx, atm in enumerate(automatas):
if (uat, atm_idx) in exempts:
continue
print 'processing {0} stride{3} automata {1} from {2}'.format(
uat, atm_idx + 1, len(automatas), stride_val)
bc_bits_len = 8
if use_compression:
bc_sym_dict = get_equivalent_symbols([atm], replace=True)
bc_bits_len = int(
math.ceil(math.log(max(bc_sym_dict.values()), 2)))
translation_list = []
for s in range(stride_val):
atm = atm.get_single_stride_graph()
if use_compression and s < compression_depth:
new_translation = get_equivalent_symbols([atm],
replace=True)
translation_list.append(new_translation)
if atm.is_homogeneous is False:
atm.make_homogenous()
minimize_automata(atm)
#lut_bram_dic = {n: (1, 2) for n in atm.nodes}
generator_ins.register_automata(
atm=atm,
use_compression=use_compression,
byte_trans_map=bc_sym_dict if use_compression else None)
if use_compression:
generator_ins.register_compressor(
[atm.id],
byte_trans_map=bc_sym_dict,
translation_list=translation_list)
if (atm_idx + 1) % atms_per_stage == 0:
generator_ins.register_stage_pending(use_bram=use_bram)
generator_ins.finilize()
draw_individually = False
max_stride = 0
switch_size = 128
routing_template = None
for anml in [AnmalZoo.TCP, AnmalZoo.ClamAV]:
if not os.path.exists("Results/" + str(anml)):
os.makedirs("Results/" +
str(anml)) #make directory if it does not exist
else:
pass
#shutil.rmtree("Results/"+str(anml)) #remove content
#os.makedirs("Results/"+str(anml)) # make directory if it does not exist
automata = atma.parse_anml_file(anml_path[anml])
automata.remove_ors()
#utility.minimize_automata(automata)
# accumulative switch map
ccs = automata.get_connected_components_as_automatas()
for stride in range(max_stride,
max_stride + 1): # one more for the original automata
acc_switch_map = np.zeros((switch_size, switch_size))
print "starting stride ", stride
for cc_idx, cc in enumerate(ccs[:]):
print "processing {} , id {}".format(anml, cc_idx)
print cc.get_summary(logo="original")
for _ in range(stride):
import automata as atma
from automata.automata_network import compare_input, get_bit_automaton, get_strided_automata2
from automata.AnmalZoo.anml_zoo import anml_path, AnmalZoo
from automata.utility.utility import minimize_automata, get_equivalent_symbols, get_unified_symbol
import math
from automata.HDL.hdl_generator import test_compressor
import logging
logging.basicConfig(level=logging.DEBUG)
automatas = atma.parse_anml_file(anml_path[AnmalZoo.Levenshtein])
automatas.remove_ors()
get_unified_symbol(atm=automatas, is_input_homogeneous=True, replace=True)
automatas = automatas.get_connected_components_as_automatas()
#automatas=pickle.load(open('Snort1-50.pkl', 'rb'))
org_atm = automatas[0]
print org_atm.get_summary(logo='original')
org_atm.draw_graph('original.svg')
bit_atm=get_bit_automaton(atm=org_atm, original_bit_width=3)
#bit_atm.draw_graph('bitwise.svg')
print bit_atm.get_summary(logo='bitwise')
strided_b_atm=get_strided_automata2(atm=bit_atm, stride_value=8, is_scalar=True, base_value=2, add_residual=False)
print strided_b_atm.get_summary(logo='strided bitwise')
#strided_b_atm.draw_graph('strided.svg')
strided_b_atm.make_homogenous()
print strided_b_atm.get_summary(logo='homogeneous')
from automata.utility.utility import minimize_automata
# Get the usage string
def usage():
usage = "----------------- Usage ----------------\n"
usage += "./count_explicit_states.py <ANML File>"
return usage
# Entry point
if __name__ == '__main__':
# Check the correct number of command line arguments
if len(sys.argv) != 2:
print(usage())
exit(-1)
# Grab the input filename
anml_input = sys.argv[1]
# Parse the ANML file
automata = atma.parse_anml_file(anml_input)
before = automata.nodes_count
# Minimizing the automata with NFA heuristics
minimize_automata(automata)
after = automata.nodes_count
print(str(before) + ", " + str(after))
def process_single_ds(uat):
#uat = AnmalZoo.Ranges05
return_result = {}
result_dir = out_dir + str(uat)
shutil.rmtree(result_dir, ignore_errors=True)
os.mkdir(result_dir)
exempts = {(AnmalZoo.Snort, 1411)}
max_target_stride = 2
uat_count = 200
automatas = atma.parse_anml_file(anml_path[uat])
automatas.remove_ors()
automatas = automatas.get_connected_components_as_automatas()
#uat_count = len(automatas) # comment this to test a subset of automatons defined in uat_count
automatas = automatas[:uat_count]
uat_count = len(automatas)
filed_names = ['number_of_states', 'number_of_edges', 'max_fan_in', 'max_fan_out',
'max_symbol_len', 'min_symbol_len', 'total_sym']
for hom_between, is_Bram in [(False, False), (False, True)]:
for stride_val in range(max_target_stride + 1):
n_states = 0.0
n_edges = 0.0
max_fan_in = 0.0
max_fan_out = 0.0
max_sym_len = 0.0
min_sym_len = 0.0
total_sym = 0.0
with open(result_dir + '/S' + str(stride_val) + '_' + str(uat_count) +
'is_HNH' + str(hom_between) + 'is_Bram' + str(is_Bram) + 'len' + str(len(automatas)) +'.csv', 'w') as csv_file:
csv_writer = csv.writer(csv_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
csv_writer.writerow(filed_names)
for atm_idx, atm in enumerate(automatas):
if (uat, atm_idx) in exempts:
continue
print 'processing {0} stride {3} number {1} from {2}'.format(uat, atm_idx, uat_count, stride_val)
for _ in range(stride_val):
if is_Bram is True and hom_between is True and atm.is_homogeneous is False:
atm.make_homogenous()
atm.make_parentbased_homogeneous()
atm = atm.get_single_stride_graph()
if atm.is_homogeneous is False:
atm.make_homogenous()
minimize_automata(atm)
if is_Bram is True and hom_between is False:
atm.fix_split_all()
all_nodes = filter(lambda n:n.id != 0, atm.nodes) # filter fake root
all_nodes_symbols_len_count = [len(n.symbols) for n in all_nodes]
n_s = atm.nodes_count
n_states += n_s
n_e = atm.edges_count
n_edges += n_e
m_f_i = atm.max_STE_in_degree()
max_fan_in += m_f_i
m_f_o = atm.max_STE_out_degree()
max_fan_out += m_f_o
mx_s_l = max(all_nodes_symbols_len_count)
max_sym_len += mx_s_l
mn_s_l = min(all_nodes_symbols_len_count)
min_sym_len += mn_s_l
t_s = sum(all_nodes_symbols_len_count)
total_sym += t_s
csv_writer.writerow([n_s, n_e, m_f_i, m_f_o, mx_s_l, mn_s_l, t_s])
n_states /= uat_count
n_edges /= uat_count
max_fan_in /= uat_count
max_fan_out /= uat_count
max_sym_len /= uat_count
min_sym_len /= uat_count
total_sym /= uat_count
return_result[(is_Bram, stride_val)] = (n_states,
n_edges,
max_fan_in,
max_fan_out,
max_sym_len,
min_sym_len,
total_sym)
with open(out_dir + 'summary.txt', 'a+') as f:
fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
to_w_lns =[]
to_w_lns.append(str(uat) + "L" + str(uat_count) + "S" + str(stride_val) + "BRam" + str(is_Bram) + "\n")
to_w_lns.append(" average number of states = " + str(n_states) + "\n")
to_w_lns.append(" average number of edges = " + str(n_edges) + "\n")
to_w_lns.append(" average max fan-in = " + str(max_fan_in) + "\n")
to_w_lns.append(" average max fan-out = " + str(max_fan_out) + "\n")
to_w_lns.append(" average max sym-len = " + str(max_sym_len) + "\n")
to_w_lns.append(" average min sym-len = " + str(min_sym_len) + "\n")
to_w_lns.append(" average total_sym-len = " + str(total_sym) + "\n")
f.writelines(to_w_lns)
fcntl.flock(f, fcntl.LOCK_UN)
return uat, return_result
def process_single_ds(uat):
try:
return_result = {}
result_dir = out_dir_prefix + str(uat)
shutil.rmtree(result_dir, ignore_errors=True)
os.mkdir(result_dir)
# cleaning the result folder
automata_per_stage = 50
# this is a pipelineing parameter for staging as pipeline. We usually use 50 for this parameter
automatas = atma.parse_anml_file(anml_path[uat])
automatas.remove_ors()
automatas = automatas.get_connected_components_as_automatas()
uat_count = 100 # number of automata to be processed
# uat_count = len(automatas) # comment this to test a subset of automatons defined in uat_count
automatas = automatas[:uat_count]
uat_count = len(automatas)
number_of_stages = math.ceil(
len(automatas) / float(automata_per_stage))
# number of pipleine stages
atms_per_stage = int(
math.ceil(len(automatas) / float(number_of_stages)))
hdl_folder_name = hd_gen.get_hdl_folder_name(
prefix=str(uat),
number_of_atms=len(automatas),
stride_value=0,
before_match_reg=False,
after_match_reg=False,
ste_type=1,
use_bram=False,
use_compression=False,
compression_depth=-1)
print "folder name to store the HDLs:", hdl_folder_name
generator_ins = hd_gen.HDL_Gen(path=os.path.join(
result_dir, hdl_folder_name),
before_match_reg=False,
after_match_reg=False,
ste_type=1,
total_input_len=dbw)
for atm_idx, atm in enumerate(automatas):
print 'processing {0} number {1} from {2}'.format(
uat, atm_idx + 1, uat_count)
b_atm = atma.automata_network.get_bit_automaton(
atm, original_bit_width=atm.max_val_dim_bits_len)
atm = atma.automata_network.get_strided_automata2(
atm=b_atm,
stride_value=dbw,
is_scalar=True,
base_value=2,
add_residual=True)
atm.make_homogenous()
minimize_automata(atm)
generator_ins.register_automata(atm=atm, use_compression=False)
if (atm_idx + 1) % atms_per_stage == 0:
generator_ins.register_stage_pending(use_bram=False)
generator_ins.finilize()
shutil.make_archive(hdl_folder_name, 'zip', result_dir)
shutil.rmtree(result_dir)
return uat, return_result
except Exception as ex:
print ex
raise ex
import automata as atma
from automata.AnmalZoo.anml_zoo import anml_path, AnmalZoo
from tqdm import tqdm
import pickle
from automata.utility.utility import minimize_automata
for automata_name, automata_path in anml_path.iteritems():
if automata_name != AnmalZoo.Levenshtein:
continue
automata = atma.parse_anml_file(automata_path)
print "Finished processing from anml file. Here is the summary", str(automata_name)
automata.remove_ors()
orig_atm_nodes_count = []
fst_stride_nodes_count = []
sec_stride_nodes_count = []
third_stride_nodes_count = []
orig_automatas = automata.get_connected_components_as_automatas()
for orig_cc in tqdm(orig_automatas, unit="automata"):
orig_atm_nodes_count.append((orig_cc.get_number_of_nodes(), orig_cc.edges_count,
orig_cc.max_STE_out_degree(),orig_cc.max_STE_in_degree() ))
orig_cc.remove_all_start_nodes()
fst_st_atm = orig_cc.get_single_stride_graph()
fst_st_atm.make_homogenous()
