def test_simple_language(self):
# Language = {a^nb : n >= 0}
m = {
"states": ["q0", "q1", "q2"],
"alphabet": ["a", "b"],
"initial": "q0",
"terminals": ["q1"],
"delta": {
("q0", "a"): "q0",
("q0", "b"): "q1",
("q1", "a"): "q2",
("q1", "b"): "q2",
("q2", "a"): "q2",
("q2", "b"): "q2",
},
}
self.my_dfa = DFA()
self.my_dfa.load(m)
self.assertTrue(self.my_dfa._accepts("ab"))
self.assertTrue(self.my_dfa._accepts("aaaab"))
self.assertTrue(self.my_dfa._accepts("b"))
self.assertFalse(self.my_dfa._accepts("aba"))
self.assertFalse(self.my_dfa._accepts("aabbabaa"))
self.assertFalse(self.my_dfa._accepts("aaaaa"))
def test_load_from_yaml(self):
self.my_dfa = DFA()
self.my_dfa.load_from_yaml("example.yaml")
self.assertTrue(self.my_dfa._accepts("ab"))
self.assertTrue(self.my_dfa._accepts("aaaab"))
self.assertTrue(self.my_dfa._accepts("b"))
self.assertFalse(self.my_dfa._accepts("aba"))
self.assertFalse(self.my_dfa._accepts("aabbabaa"))
self.assertFalse(self.my_dfa._accepts("aaaaa"))
def buildMinDFA(dfa, partition):
## showPartition(partition)
## print [s.name for s in dfa.listStates()]
## First make a new DFA and ensure that its start state is the same
## as that of the original DFA. For all the other states in the
## new DFA, just select a state from each of the state sets in the
## partition (expect for any dead-state state sets, which are
## ignored).
newDFA = DFA()
newDFA.startState = dfa.startState
newDFA.alphabet = dfa.alphabet
selectedStates = []
for stateSet in partition:
if newDFA.startState in stateSet:
selectedStates.append((newDFA.startState,stateSet))
elif stateSet != deadSS:
selectedStates.append((list(stateSet)[0],stateSet))
## Now construct new links in the new DFA. Note that we are
## modifiying states *shared* with the original DFA, so this
## action *corrupts* the original DFA.
newDFA.finalStates = []
newDFA.regExprs = []
for state,stateSet in selectedStates:
for i,successor in enumerate(state.successors):
for targetState,targetStateSet in selectedStates:
if successor[1] in targetStateSet:
state.successors[i] = (successor[0],targetState)
break
if state in dfa.finalStates:
newDFA.finalStates.append(state)
newDFA.regExprs.append(dfa.regExprs[dfa.finalStates.index(state)])
newDFA.stateCount = len(selectedStates)
return newDFA
def __gen_states__(self):
'''
ref: Definition 4.0.18
:return:
'''
# 首先是基底
self._states = set()
self._states.add(((0,0),))
ans = set()
for i in xrange(self._r): #base position loop
# 以i为基得state
base_state = DFA.triangle_area(i, 0, self._k, self._w)
#print "base_state %d: " % i , base_state
def handler(pos_e_pair):
pos_e_pair = list(pos_e_pair)
i, e = pos_e_pair[0]
for j, f in pos_e_pair[1:]:
if abs(j - i) <= abs(f - e): # subsume
return False
return True
tmp = setUtil.subset(list(base_state), handler)
ans.update(tmp)
self._states.update(DFA.i_unique(ans))
self._states = list(self._states)
def dfa2aig(dfa: DFA):
"""
Takes a dfa.DFA object and returns a tuple of:
1. An aiger_bv.AIGBV circuit modeling the DFA's labeling and
transitions. This circuit has 1 input, output, and latch
called "action", "output", and "state" respectively.
2. A dictionary with at three entries, "inputs", "outputs", and
"states".
- Each entry of this dictionary is a bidict that maps one-hot
encoded tuples, e.g. (True, False, False), to dfa inputs,
outputs, states.
3. An aiger_bv.AIGBV circuit which monitors is all inputs are
valid, e.g., one_hot encoded.
"""
in2bv, out2bv, state2bv = create_bv_maps(dfa)
dfa_dict, _ = dfa2dict(dfa)
action = aiger_bv.atom(len(dfa.inputs), 'action', signed=False)
state = aiger_bv.atom(len(dfa.states()), 'state', signed=False)
circ = out_circ(dfa_dict, state2bv, out2bv, state)
circ |= transition_circ(dfa_dict, state2bv, in2bv, action, state)
start = state2bv[dfa.start]
circ = circ.loopback({
"input": "state",
"output": "next_state",
"init": start,
"keep_output": False,
})
relabels = {
'inputs': in2bv.inv,
'outputs': out2bv.inv,
'states': state2bv.inv
}
return circ, relabels, valid_circ(action)
def to_dfa(self):
"""
The algorithm used here to transform the NFA into a DFA is as follows:
(1) Qd = {q0n} add initial state from NFA into set of states Q of new DFA
(2) for each state in Qd find the resulting state for each symbol from the alphabet using the transition function
(3) add the new resulting state into Qd then repeat the process
(4) Once all the states have been checked, the algorithm is finished
(5) Each new state that contains a final state is itself a final state of the resulting DFA
"""
# since a list is not hashable (can not be used as key in a dictionary)
# I am casting it to a frozenset, which is a hashable datatype
q0 = frozenset(self.init_states)
Qnew = set([q0]) # Qd is called Qnew
unchecked = Qnew.copy() # make a copy of Qd
delta_new = dict() # delta is a dictionary
final_states = []
sigma_new = self.sigma
while len(unchecked) > 0:
curr = unchecked.pop()
for sym in sigma_new:
try:
# obtain the next states using the transition function
# new state into hashable frozenset, and add to new transition function and Qnew
next_states = reduce(lambda x,y: x|y, [frozenset(self.transition_func[(q,sym)]) for q in curr if (q,sym) in self.transition_func.keys()])
next_states = frozenset(next_states)
delta_new[(curr,sym)] = next_states
if next_states not in Qnew:
Qnew.add(next_states)
unchecked.add(next_states)
# catch any possible exceptions
except Exception:
continue
for curr in Qnew:
# check which states in the new DFA contain any final states
# from the NFA, and add the ones that do to the set of final_states
if len(curr & frozenset(self.final_states)) > 0:
final_states.append(curr)
return DFA(self.sigma, Qnew, q0, delta_new, final_states)
def __switching_transitions(delf, dfa):
new_initial = 'q0'
new_states = deepcopy(dfa.states)
new_accepting = deepcopy(dfa.accepting)
# Changing transitions
def switch(edge_label):
return '0' if edge_label == '1' else (
'1' if edge_label == '0' else edge_label)
new_transitions = {
state: {
switch(edge_label): dfa.transitions[state][edge_label]
for edge_label in dfa.transitions[state]
}
for state in new_states - {'qF'}
}
new_dfa = DFA(new_states, new_transitions, new_initial, new_accepting)
info(new_dfa)
return new_dfa
def toDFA(self):
from dfa import DFA, DTransitionFunction
if self.func.epsilon:
return self.toNFA().toDFA()
def rename_state(state):
import re
return re.sub(r"[\{\}\,\s]", r"", f"[{state}]")
dfa_δ = DTransitionFunction(self.alphabet)
states = [set([self.initial_state])]
index = 0
while index < len(states):
for letter in self.alphabet:
res = set()
for state in states[index]:
res = res | self.func.get((state, letter), set())
state_name = str(
states[index]) if len(states[index]) != 0 else 'Ø'
dfa_δ[state_name, letter] = str(res) if len(res) != 0 else 'Ø'
if res not in states:
states.append(res)
index += 1
seen = set()
Q = [rename_state(key[0]) for key in dfa_δ]
Q = [state for state in Q if not (state in seen or seen.add(state))]
δ = DTransitionFunction(
self.alphabet, {(rename_state(key[0]), key[1]): rename_state(value)
for key, value in dfa_δ.items()})
initial_state = Q[Q.index(f"[{self.initial_state}]")]
accepted_sates = set(
rename_state(state) for accepting in self.accepted_sates
for state in states if accepting in state)
return DFA(Q, self.alphabet, δ, initial_state, accepted_sates)
def determinize(self):
states = set(self.transitions.keys())
actions = set()
for state, out_transitions in self.transitions.items():
actions.update(out_transitions.keys())
dfa_start = frozenset([self.start])
dfa_states_left = set([dfa_start])
mappings = {dfa_start: 1}
dfa_transitions = {}
while len(dfa_states_left) > 0:
dfa_state = dfa_states_left.pop()
dfa_transitions[mappings[dfa_state]] = {}
for action in actions:
dfa_next_state = set()
for nfa_state in dfa_state:
try:
dfa_next_state.update(
self.transitions[nfa_state][action])
except KeyError:
pass
dfa_next_state = frozenset(dfa_next_state)
if dfa_next_state not in mappings:
mappings[dfa_next_state] = len(mappings) + 1
dfa_states_left.add(dfa_next_state)
dfa_transitions[
mappings[dfa_state]][action] = mappings[dfa_next_state]
dfa_accepting = [
mappings[dfa_state] for dfa_state in mappings.keys()
if not dfa_state.isdisjoint(self.accepting)
]
return DFA(transitions=dfa_transitions,
start=1,
accepting=dfa_accepting)
def rand_benchmark(save_dir=None):
dfa = DFA(0, {0}, {0: {0: 0}})
full_alphabet = "abcdefghijklmnopqrstuvwxyz"
alphabet = full_alphabet[0:5]
benchmark = {}
benchmark.update({"alph_len": len(alphabet)})
while len(dfa.states) < 5:
max_final_states = np.random.randint(5, 29)
dfa_rand1 = random_dfa(alphabet, min_states=max_final_states, max_states=30, min_final=1,
max_final=max_final_states)
dfa = minimize_dfa(dfa_rand1)
benchmark.update({"dfa_states": len(dfa.states), "dfa_final": len(dfa.final_states)})
print("DFA to learn {}".format(dfa))
learn_dfa_with_rnn(dfa, benchmark, save_dir)
return benchmark
def from_dfa_to_sup_dfa_gen(dfa: DFA, tries=5):
not_final_states = [
state for state in dfa.states if state not in dfa.final_states
]
if len(not_final_states) == 1:
return
created_dfas = []
for _ in range(tries):
s = np.random.randint(1, len(not_final_states))
new_final_num = np.random.choice(len(not_final_states),
size=s,
replace=False)
new_final = [not_final_states[i] for i in new_final_num]
dfa_spec = DFA(dfa.init_state, dfa.final_states + new_final,
dfa.transitions)
dfa_spec = minimize_dfa(dfa_spec)
if dfa_spec in created_dfas:
continue
created_dfas.append(dfa_spec)
yield dfa_spec
def _produce_hypothesis_set(self):
"""
Like regular produce_hypothesis but done for a batches of words.
This is a speeding up for RNN learning.
"""
transitions = {}
final_nodes = []
leafs_plus_letters = []
for leaf in self._leafs:
if leaf.inLan:
final_nodes.append(leaf.name)
for letter in self.teacher.alphabet:
leafs_plus_letters.append(leaf.name + tuple([letter]))
states = self._sift_set(leafs_plus_letters)
for leaf in range(len(self._leafs)):
transition = {}
for letter in range(len(self.teacher.alphabet)):
transition.update(
{self.teacher.alphabet[letter]: states[leaf * len(self.teacher.alphabet) + letter].name})
transitions.update({self._leafs[leaf].name: transition})
return DFA(tuple(""), final_nodes, transitions)
def symmetric_difference(lhs, rhs):
"""Construct the symmetric difference of two automata.
lhs and rhs are the two input automata and res is the result of this
function:
L(res) = (L(lhs) \ L(rhs)) u (L(rhs) \ L(lhs))
res can be constructed by building the product of lhs and rhs and setting
the final states as: (lhs.F x ~rhs.F) u (~lhs.F x rhs.F)
"""
def new_state_name(ls, rs):
return ls * len(rhs.states) + rs
# the automatons should have the same alphabet!
alphabet = lhs.alphabet
states = set(range(len(lhs.states) * len(rhs.states)))
final_states = set()
for ls in lhs.states:
for rs in rhs.states:
ns = new_state_name(ls, rs)
if (ls in lhs.final_states) and (rs not in rhs.final_states) or \
(ls not in lhs.final_states) and (rs in rhs.final_states):
final_states.add(ns)
start_state = new_state_name(lhs.start_state, rhs.start_state)
delta = {}
for ls in lhs.states:
for rs in rhs.states:
for ch in alphabet:
ns = new_state_name(ls, rs)
nns = new_state_name(lhs.delta[(ls, ch)], rhs.delta[(rs, ch)])
delta[(ns, ch)] = nns
return DFA(alphabet, states, start_state, final_states, delta)
def __remove_final_state(self, dfa):
"""
Returns a DFA with one less state than the given DFA
@param DFA: DFA with n states
@return new_DFA: DFA with n-1 states
"""
# Getting q_n of given DFA
prev_final_state = "q" + str(len(dfa.states) - 2)
# Removing q_n from states
new_states = dfa.states - {prev_final_state}
# Getting q_n of new DFA
new_final_state = "q" + str(len(new_states) - 2)
# Deleting transitions associated with q_n from transitions
new_transitions = deepcopy(dfa.transitions)
new_transitions[new_final_state]['0'] = new_final_state
new_transitions[new_final_state]['1'] = new_final_state
del new_transitions[prev_final_state]
# Removing prev_final_state from accepting states of new dfa, if necessary
new_accepting = dfa.accepting - {prev_final_state}
new_dfa = DFA(new_states, new_transitions, 'q0', new_accepting)
info(new_dfa)
return new_dfa
def test_tee():
machine = tee(
lift(universal({0, 1})),
lift(empty({1})),
)
assert len(machine.states()) == 1
assert machine.inputs == {1}
assert machine.outputs.left == machine.outputs.right == {False, True}
assert machine.label((1, 1, 1)) == (True, False)
identity = lift(
DFA(
start=1,
label=lambda x: x,
transition=lambda _, c: c,
outputs={1},
inputs={1},
))
machine = identity >> machine
assert len(machine.states()) == 1
assert machine.inputs == {1}
assert machine.label((1, 1, 1)) == (True, False)
def _dfa(self):
"""Extracts the DFA from the given enumeration state."""
A = characters('a', self.k)
Q = characters('0', self.n)
d = []
for i in range(self.n):
for j in range(self.k):
state1 = chr(ord('0') + i )
symbol = chr(ord('a') + j )
state2 = chr(ord('0') + self.stateDelta[i * self.k + j])
d.append(((state1, symbol), state2))
F = []
for i in range(self.n):
if self.stateF[i]:
F.append(chr(ord('0') + i))
return DFA(A, Q, d, '0', F, self.k, self.n, self.f)
def rand_min_dfa(k, n, f, dmin, dmax, planar, outDir):
"""Tries to generate a random minimal DFA matching some properties.
Uses the DB mentioned in the module description.
- all DFAs generated here and by 'next_min_dfa' have different languages
- ensures properties checked by _valid_dfa
- matching DFAs are saved in the DB at table 'MinimalDFAs'
"""
dbConn = sqlite3.connect(outDir / _DB_LOC)
db_dfa.ensure_validity(dbConn)
matchingUsedDFAs = db_dfa.fetch(dbConn, k, n, f)
A = characters('a', k)
Q = characters('0', n)
while True:
testDFA = DFA(A, Q, [], '0', random.sample(Q, f), k, n, f)
for q in testDFA.states:
for c in testDFA.alphabet:
p = random.choice(testDFA.states)
testDFA.transitions.append(((q, c), p))
# _valid_dfa sets testDFA.depth and testDFA.planar
if _valid_dfa(testDFA, dmin, dmax, planar, matchingUsedDFAs):
# save needs testDFA.depth and testDFA.planar
db_dfa.save(dbConn, testDFA)
dbConn.close()
return testDFA
class Brzozowski:
"""
This class transforms a pyfst DFA to regular expressions
using the Brzozowski Algebraic Method
"""
def __init__(self, input_fst_a, alphabet=None):
"""
Args:
input_fst_a (DFA): The DFA states
alphabet (list): the input alphabet
Returns:
None
"""
if alphabet is None:
alphabet = createalphabet()
self.alphabet = alphabet
self.mma = DFA(self.alphabet)
self.mma.init_from_acceptor(input_fst_a)
# a is a matrix that holds all transitions
# If there is a transition from state 0 to state 1 with the symbol x
# then a[0][1]=x
self.A = {}
# B[n] holds the regular expression that describes how a final state
# can be reached from state n
self.B = {}
self.epsilon = ''
self.empty = None
def _bfs_sort(self, start):
"""
maintain a map of states distance using BFS
Args:
start (fst state): The initial DFA state
Returns:
list: An ordered list of DFA states
using path distance
"""
pathstates = {}
# maintain a queue of nodes to be visited. Both current and previous
# node must be included.
queue = []
# push the first path into the queue
queue.append([0, start])
pathstates[start.stateid] = 0
while queue:
# get the first node from the queue
leaf = queue.pop(0)
node = leaf[1]
pathlen = leaf[0]
# enumerate all adjacent nodes, construct a new path and push it
# into the queue
for arc in node.arcs:
next_state = self.mma[arc.nextstate]
if next_state.stateid not in pathstates:
queue.append([pathlen + 1, next_state])
pathstates[next_state.stateid] = pathlen + 1
orderedstatesdict = OrderedDict(
sorted(pathstates.items(), key=lambda x: x[1], reverse=False))
for state in self.mma.states:
orderedstatesdict[state.stateid] = state
orderedstates = [x[1] for x in list(orderedstatesdict.items())]
return orderedstates
def star(self, input_string):
"""
Kleene star operation
Args:
input_string (str): The string that the kleene star will be made
Returns:
str: The applied Kleene star operation on the input string
"""
if input_string != self.epsilon and input_string != self.empty:
return "(" + input_string + ")*"
else:
return ""
def _brzozowski_algebraic_method_init(self):
"""Initialize Brzozowski Algebraic Method"""
# Initialize B
for state_a in self.mma.states:
if state_a.final:
self.B[state_a.stateid] = self.epsilon
else:
self.B[state_a.stateid] = self.empty
# Initialize A
for state_b in self.mma.states:
self.A[state_a.stateid, state_b.stateid] = self.empty
for arc in state_a.arcs:
if arc.nextstate == state_b.stateid:
self.A[state_a.stateid, state_b.stateid] = \
self.mma.isyms.find(arc.ilabel)
def _brzozowski_algebraic_method_solve(self):
"""Perform Brzozowski Algebraic Method"""
orderedstates = self._bfs_sort(
sorted(self.mma.states, key=attrgetter('initial'),
reverse=True)[0])
for n in range(len(orderedstates) - 1, 0, -1):
# print "n:" + repr(n)
if self.A[orderedstates[n].stateid,
orderedstates[n].stateid] != self.empty:
# B[n] := star(A[n,n]) . B[n]
if self.B[orderedstates[n].stateid] != self.empty:
self.B[orderedstates[n].stateid] = \
self.star(self.A[orderedstates[n].stateid, orderedstates[n].stateid]) \
+ self.B[orderedstates[n].stateid]
else:
self.B[orderedstates[n].stateid] = self.star(
self.A[orderedstates[n].stateid,
orderedstates[n].stateid])
for j in range(0, n):
# A[n,j] := star(A[n,n]) . A[n,j]
if self.A[orderedstates[n].stateid,
orderedstates[j].stateid] != self.empty:
self.A[
orderedstates[n].stateid,
orderedstates[j].stateid] = \
self.star(self.A[orderedstates[n].stateid, orderedstates[n].stateid]) \
+ self.A[orderedstates[n].stateid, orderedstates[j].stateid]
else:
self.A[orderedstates[n].stateid,
orderedstates[j].stateid] = self.star(
self.A[orderedstates[n].stateid,
orderedstates[n].stateid])
for i in range(0, n):
# B[i] += A[i,n] . B[n]
newnode = None
if self.A[orderedstates[i].stateid, orderedstates[n].stateid] != self.empty \
and self.B[orderedstates[n].stateid] != self.empty:
newnode = self.A[orderedstates[i].stateid,
orderedstates[n].stateid] + self.B[
orderedstates[n].stateid]
elif self.A[orderedstates[i].stateid,
orderedstates[n].stateid] != self.empty:
newnode = self.A[orderedstates[i].stateid,
orderedstates[n].stateid]
elif self.B[orderedstates[n].stateid] != self.empty:
newnode = self.B[orderedstates[n].stateid]
if self.B[orderedstates[i].stateid] != self.empty:
if newnode is not None:
self.B[orderedstates[i].stateid] += newnode
else:
self.B[orderedstates[i].stateid] = newnode
for j in range(0, n):
# A[i,j] += A[i,n] . A[n,j]
newnode = None
if self.A[
orderedstates[i].stateid,
orderedstates[n].stateid] != self.empty \
and self.A[orderedstates[n].stateid, orderedstates[j].stateid] \
!= self.empty:
newnode = self.A[orderedstates[i].stateid,
orderedstates[n].stateid] + self.A[
orderedstates[n].stateid,
orderedstates[j].stateid]
elif self.A[orderedstates[i].stateid,
orderedstates[n].stateid] != self.empty:
newnode = self.A[orderedstates[i].stateid,
orderedstates[n].stateid]
elif self.A[orderedstates[n].stateid,
orderedstates[j].stateid] != self.empty:
newnode = self.A[orderedstates[n].stateid,
orderedstates[j].stateid]
if self.A[orderedstates[i].stateid,
orderedstates[j].stateid] != self.empty:
if newnode is not None:
self.A[orderedstates[i].stateid,
orderedstates[j].stateid] += newnode
else:
self.A[orderedstates[i].stateid,
orderedstates[j].stateid] = newnode
def get_regex(self):
"""Generate regular expressions from DFA automaton"""
self._brzozowski_algebraic_method_init()
self._brzozowski_algebraic_method_solve()
return self.B
from itertools import product
from dfa import DFA
from probabilistic_automata import pa as PA
PARITY = DFA(
start=0,
inputs={0, 1},
label=bool,
transition=lambda s, c: (s + c) & 1,
)
def test_uniform_dist_smoke():
actions = {1, 2, 3}
dist = PA.uniform(actions)(0, 1)
assert dist.sample() in actions
for a in actions:
assert dist(a) == 1 / 3
def test_lift():
pdfa = PA.lift(PARITY)
assert pdfa.inputs == PARITY.inputs
assert pdfa.env_inputs == {None}
assert pdfa.outputs == {0, 1}
for s, a in product(pdfa.states(), pdfa.inputs):
import pytest
from dfa import DFA
from probabilistic_automata import pa as PA
from probabilistic_automata import markov_chain as MC
SYS = DFA(
start=0,
inputs={0, 1},
label=lambda s: s == 2,
transition=lambda s, c: (s + c) % 4,
)
def test_validator():
with pytest.raises(ValueError):
MC.from_pdfa(PA.lift(SYS)) # Non-determinism.
MC.from_pdfa(PA.randomize(SYS)) # Actually Markov Chain.
def test_trace_prob():
mc1 = MC.from_pdfa(PA.randomize(SYS))
trc = 3 * [0]
assert pytest.approx(2**-3) == mc1.trace_prob(trc)
trc = 3 * [1, 2, 1]
assert 0 == mc1.trace_prob(trc)
class DFATest(unittest.TestCase):
def test_simple_language(self):
#Language = {a^nb : n >= 0}
m = {
'states': ["q0", "q1", "q2"],
'alphabet': ["a", "b"],
'initial': "q0",
'terminals': ["q1"],
'delta': {
("q0", "a"): "q0",
("q0", "b"): "q1",
("q1", "a"): "q2",
("q1", "b"): "q2",
("q2", "a"): "q2",
("q2", "b"): "q2"
}
}
self.my_dfa = DFA()
self.my_dfa.load(m)
self.assertTrue(self.my_dfa._accepts("ab"))
self.assertTrue(self.my_dfa._accepts("aaaab"))
self.assertTrue(self.my_dfa._accepts("b"))
self.assertFalse(self.my_dfa._accepts("aba"))
self.assertFalse(self.my_dfa._accepts("aabbabaa"))
self.assertFalse(self.my_dfa._accepts("aaaaa"))
def test_load_from_yaml(self):
self.my_dfa = DFA()
self.my_dfa.load_from_yaml("example.yaml")
self.assertTrue(self.my_dfa._accepts("ab"))
self.assertTrue(self.my_dfa._accepts("aaaab"))
self.assertTrue(self.my_dfa._accepts("b"))
self.assertFalse(self.my_dfa._accepts("aba"))
self.assertFalse(self.my_dfa._accepts("aabbabaa"))
self.assertFalse(self.my_dfa._accepts("aaaaa"))
def test_1():
sensitive_path = 'sensitive/sensitive.txt'
dfa = DFA()
dfa.load_sensitive(sensitive_path)
dfa = DFA()
#dfa.load_from_yaml('automata/mydata.yaml')
dfa.load_from_pickle('automata/mydata.json')
dfa.accepts('我爱北京天安门\n')
dfa.accepts('我喜欢做假证\n')
dfa.accepts('我喜欢做嗳\n')
with open(sensitive_path, 'r', encoding='UTF-8') as f:
lines = f.readlines()
result = []
lines = [i for i in lines if i != '\n']
for i in lines:
i = '我喜欢' + i
print(i)
dfa.accepts(i)
start_time = time.time()
for i in lines:
i = '我喜欢' + i
result.append(dfa.predict(i))
during_time = time.time() - start_time
print(r'耗时:', during_time)
print(r'平均z耗时:', during_time / len(lines))
result = np.array(result)
print(r'正确个数:', len(result[result == True]))
print(r'错误个数:', len(result[result == False]))
print(r'正确率:', len(result[result == True]) / len(result))
class Regex:
"""
This class transforms a pyfst DFA to regular expressions
using the Brzozowski Algebraic Method
"""
def __init__(self, input_fst_a, alphabet=None):
"""
Args:
input_fst_a (DFA): The DFA states
alphabet (list): the input alphabet
Returns:
None
"""
if alphabet is None:
alphabet = createalphabet()
self.alphabet = alphabet
self.mma = DFA(self.alphabet)
self.mma.init_from_acceptor(input_fst_a)
# a is a matrix that holds all transitions
# If there is a transition from state 0 to state 1 with the symbol x
# then a[0][1]=x
self.A = {}
# B[n] holds the regular expression that describes how a final state
# can be reached from state n
self.B = {}
self.epsilon = ''
self.empty = None
def _bfs_sort(self, start):
"""
maintain a map of states distance using BFS
Args:
start (fst state): The initial DFA state
Returns:
list: An ordered list of DFA states
using path distance
"""
pathstates = {}
# maintain a queue of nodes to be visited. Both current and previous
# node must be included.
queue = []
# push the first path into the queue
queue.append([0, start])
pathstates[start.stateid] = 0
while queue:
# get the first node from the queue
leaf = queue.pop(0)
node = leaf[1]
pathlen = leaf[0]
# enumerate all adjacent nodes, construct a new path and push it
# into the queue
for arc in node.arcs:
next_state = self.mma[arc.nextstate]
if next_state.stateid not in pathstates:
queue.append([pathlen + 1, next_state])
pathstates[next_state.stateid] = pathlen + 1
orderedstatesdict = OrderedDict(
sorted(pathstates.items(), key=lambda x: x[1], reverse=False))
for state in self.mma.states:
orderedstatesdict[state.stateid] = state
orderedstates = [x[1] for x in list(orderedstatesdict.items())]
return orderedstates
def star(self, input_string):
"""
Kleene star operation
Args:
input_string (str): The string that the kleene star will be made
Returns:
str: The applied Kleene star operation on the input string
"""
if input_string != self.epsilon and input_string != self.empty:
return "(" + input_string + ")*"
else:
return ""
def _create_map(self):
"""Initialize Brzozowski Algebraic Method"""
# at state i is represented by the regex self.B[i]
for state_a in self.mma.states:
self.A[state_a.stateid] = {}
# Create a map state to state, with the transition symbols
for arc in state_a.arcs:
if arc.nextstate in self.A[state_a.stateid]:
self.A[state_a.stateid][arc.nextstate].append(
self.mma.isyms.find(arc.ilabel))
else:
self.A[state_a.stateid][arc.nextstate] = [
self.mma.isyms.find(arc.ilabel)
]
if state_a.final:
self.A[state_a.stateid]['string'] = ['']
# In both self.B and Self.A there are empty spots that we will fill in _brzozowski_algebraic_method_solve
def _fix_pattern(self):
for state_a in self.mma.states:
for target in self.A[state_a.stateid]:
if len(self.A[state_a.stateid][target]) == 0:
self.A[state_a.stateid][target] = ''
elif len(self.A[state_a.stateid][target]) == 1:
self.A[state_a.stateid][target] = self.A[
state_a.stateid][target][0]
elif len(self.A[state_a.stateid][target]) == len(
self.alphabet):
self.A[state_a.stateid][target] = '.'
elif len(
[a for a in self.A[state_a.stateid][target] if len(a) > 1
]) == 0 and len(self.A[state_a.stateid][target]) > len(
self.alphabet) / 2:
self.A[state_a.stateid][target] = '[^' + ''.join(
filter(
lambda x: x not in self.A[state_a.stateid][target],
self.alphabet)) + ']'
elif len(
[a for a in self.A[state_a.stateid][target] if len(a) > 1
]) == 0 and len(self.A[state_a.stateid][target]) < len(
self.alphabet) / 2:
self.A[state_a.stateid][target] = '[' + ''.join(
self.A[state_a.stateid][target]) + ']'
elif len(
[a for a in self.A[state_a.stateid][target] if len(a) > 1
]) > 0:
self.A[state_a.stateid][target] = '(' + '|'.join(
self.A[state_a.stateid][target]) + ')'
def partitionAlphabet(self, alphabetLen, parsedInput):
intoInput = []
notInInput = []
for c in self.alphabet:
if c in parsedInput:
intoInput.append(c)
else:
notInInput.append(c)
if len(intoInput) == alphabetLen:
return '.'
if (len(intoInput) < len(notInInput)):
final = '[' + ''.join(intoInput) + ']'
else:
final = '[^' + ''.join(notInInput) + ']'
return final
def replaceAlphabet(self, input):
if "|" not in input:
return input
alphabetLen = len(self.alphabet)
parsedInput = input.split("|")
if "|||" in input:
parsedInput.append("|")
for c in parsedInput:
if len(c) > 1:
return input
final = self.partitionAlphabet(alphabetLen, parsedInput)
return final
def cleaner(self, res):
res = self.replaceAlphabet(res)
import re
if '|' in re.sub('\(.*\)', '', res):
res = '(' + res + ')'
return res
def cleanerS(self, res):
res = self.replaceAlphabet(res)
import re
if '|' in re.sub(r'\([^)]*\)', '', res):
res = '(' + res + ')*'
return res
def findWhichSymbolsFromOtherStatesConnectToMe(self, state):
array = []
for n in self.A:
if n != state and state in self.A[n]:
if self.A[n][state] not in array:
array.append(self.A[n][state])
return array
def checkIfSameStateNotNeeded(self, state, same):
array = self.findWhichSymbolsFromOtherStatesConnectToMe(state)
parts = []
if len(array) == 0:
return same
try:
if len(self.A[state]['same']
) > 3 and self.A[state]['same'][0] == '(' and self.A[state][
'same'][len(self.A[state]['same']) -
1:len(self.A[state]['same']) + 1]:
parts.append(
self.A[state]['same'][1:len(self.A[state]['same']) - 2])
else:
parts = re.findall(r'\((.+?)\)\*', self.A[state]['same'])
for n in parts:
if n[len(n) - 1] != array[0]:
return same
return ''
except AttributeError:
return same
def checkIfBothAreNeeded(
self, inputA,
inputB): # based on the fact that ([^b]*|b([^i])*) is equal to .*
try:
input1 = re.search('\(\[\^(.+?)\]\)\*', inputA).group(1)
except AttributeError:
input1 = inputA
try:
input2FirstChar = re.search('\((.+?)\)\*', inputB).group(1)[0]
except AttributeError:
input2FirstChar = inputB[0]
if len(input1) == 1 and input1 == input2FirstChar:
return 1
else:
return -1
def replaceAlphabetMixed(self, input1, input2):
found1 = 0
found2 = 0
if "|" not in input1:
final1 = input1
if len(input1) > 1:
found1 = 1
else:
parsedInput = input1.split("|")
for c in parsedInput:
if len(c) > 1:
final1 = input1
found1 = 1
break
if "|" not in input2:
final2 = input2
if len(input2) > 1:
found2 = 1
else:
parsedInput = input2.split("|")
for c in parsedInput:
if len(c) > 1:
final2 = input2
found2 = 1
break
if found1 == 1 and found2 == 0:
if (self.checkIfBothAreNeeded(input1, input2) == 1):
return '(.)*'
final2 = self.replaceAlphabet(input2)
if found1 == 0 and found2 == 1:
if (self.checkIfBothAreNeeded(input1, input2) == 1):
return '(.)*'
final1 = self.replaceAlphabet(input1)
if found1 == 1 and found2 == 1:
if (self.checkIfBothAreNeeded(input1, input2) == 1):
return '(.)*'
if found1 == 0 and found2 == 0:
final1 = input1
final2 = input2
return final1 + "|" + final2
def getMaxInternalPath(self, inkeys):
maxkey = 0
keys = inkeys
if 'same' in keys:
keys.remove('same')
if 'rest' in keys:
keys.remove('rest')
if 'string' in keys:
keys.remove('string')
if len(keys) > 0:
maxkey = sorted(keys, reverse=True)[0]
return maxkey
def existsPath(self, initial, finalstate, visited):
for i in self.A:
if finalstate in self.A[i] and i not in visited:
if i == initial:
return 1, self.A[i][finalstate], finalstate
else:
visited.append(i)
res, key, transfer_state = self.existsPath(
initial, i, visited)
if res == 1:
return 1, key, finalstate
else:
continue
else:
continue
return 0, 0, 0
def fixBrzozowskiBackwardLoopRemoval(self):
# Remove targets that point to start
for i in self.A:
if i != 0:
for target in self.A[i].keys():
if target == 0:
self.A[i].pop(target)
for i in self.A:
poplist = []
for target in self.A[i]:
if target < i:
exists, transfer_value, transfer_state = self.existsPath(
target, i, [])
if exists and (
transfer_value == self.A[i][target] or
(target in self.A[target] and
(self.A[target][target] == self.A[i][target] or len(
set(self.A[i][target]) -
set(self.A[target][target])) == 0))):
poplist.append(target)
for k in poplist:
self.A[i].pop(k)
def _Lexical(self):
found = 0
start = 0
# First Try to Optimize the Strings by doing the possible replacements of special cases to string. e.c change a loop with character e to (e)*
for state_a in self.A.keys():
# Loopholes
if state_a in self.A[state_a]:
same = ''
if 'same' in self.A[state_a]:
same = '|' + self.A[state_a]['same']
self.A[state_a]['same'] = '(' + ''.join(
self.replaceAlphabet(
self.A[state_a][state_a])) + ')*' + same
del (self.A[state_a][state_a])
# Final State
if len(self.A[state_a].keys()
) == 1 and 'string' in self.A[state_a].keys():
for state_b in self.A:
if state_b != state_a and state_b in self.A:
if state_a in self.A[state_b]:
if 'rest' in self.A[state_b] and self.A[state_b][
'rest'] != '' and self.A[state_b][
'rest'] == self.cleaner(
self.A[state_a]['string']):
if 'string' not in self.A[state_b]:
self.A[state_b]['string'] = "(" + self.A[
state_b][state_a] + ")?" + self.A[
state_b]['rest']
else:
self.A[state_b]['string'] = "(" + self.A[
state_b]['string'] + "|(" + self.A[
state_b][state_a] + ")?" + self.A[
state_b]['rest'] + ")"
del (self.A[state_b][state_a])
del (self.A[state_b]['rest'])
found = 1
else:
newinput = ''.join(
self.A[state_b][state_a]) + self.cleaner(
self.A[state_a]['string'])
if 'rest' in self.A[state_b] and self.A[
state_b]['rest'] != '':
newinput = '(' + newinput + '|' + self.A[
state_b]['rest'] + ')'
self.A[state_b]['rest'] = newinput
del (self.A[state_b][state_a])
found = 1
if state_a != start:
del (self.A[state_a])
if state_a != start and state_a in self.A and len(
self.A[state_a].keys(
)) == 1 and 'same' in self.A[state_a].keys():
del (self.A[state_a])
for state_b in self.A:
if state_b in self.A and state_a in self.A[state_b]:
del (self.A[state_b][state_a])
found = 1
# Clean Loophole
if state_a in self.A and len(self.A[state_a].keys(
)) == 1 and 'same' in self.A[state_a].keys():
del (self.A[state_a])
found = 1
# Final State
if state_a in self.A and len(
self.A[state_a].keys()) == 1 and 'rest' in self.A[state_a]:
self.A[state_a]['string'] = self.A[state_a]['rest']
del (self.A[state_a]['rest'])
found = 1
# Final State with alternative path
if state_a in self.A and len(self.A[state_a].keys()) == 2 and 'string' in self.A[state_a] and 'rest' in \
self.A[state_a]:
if self.A[state_a]['rest'] != '':
self.A[state_a]['string'] = self.cleaner(
self.A[state_a]
['string']) + '(' + self.replaceAlphabet(
self.A[state_a]['rest']) + ')?'
del (self.A[state_a]['rest'])
found = 1
# State with alternative path and self loop
if state_a in self.A and len(self.A[state_a].keys()) == 2 and 'same' in self.A[state_a] and 'rest' in \
self.A[state_a]:
self.A[state_a]['rest'] = self.checkIfSameStateNotNeeded(
state_a, self.cleanerS(
self.A[state_a]['same'])) + self.cleaner(
self.A[state_a]['rest'])
del (self.A[state_a]['same'])
found = 1
# Final State with self loop
if state_a in self.A and len(self.A[state_a].keys()) == 2 and 'string' in self.A[state_a] and 'same' in \
self.A[state_a]:
self.A[state_a]['string'] = self.checkIfSameStateNotNeeded(
state_a, self.cleanerS(
self.A[state_a]['same'])) + self.cleaner(
self.A[state_a]['string'])
del (self.A[state_a]['same'])
found = 1
# Final State with alternative path and self loop
if state_a in self.A and len(self.A[state_a].keys()) == 3 and 'string' in self.A[state_a] and 'rest' in \
self.A[state_a] and 'same' in self.A[state_a]:
if self.A[state_a]['rest'] != '':
add = '(' + self.A[state_a]['rest'] + ')?'
self.A[state_a]['string'] = self.checkIfSameStateNotNeeded(
state_a, self.cleanerS(
self.A[state_a]['same'])) + self.cleaner(
self.A[state_a]['string']) + add
del (self.A[state_a]['rest'])
del (self.A[state_a]['same'])
found = 1
# No path State
if state_a != start and state_a in self.A and len(
self.A[state_a].keys()) == 0:
del (self.A[state_a])
for state_b in self.A:
if state_b in self.A and state_a in self.A[state_b]:
del (self.A[state_b][state_a])
return found
def fixBrzozowskiAdvanced(self, found):
start = 0
for state_a in self.A:
maxkey = self.getMaxInternalPath(self.A[state_a].keys())
if state_a != start and state_a in self.A and maxkey < state_a and (
(len(self.A[state_a].keys()) > 1) or
(len(self.A[state_a].keys()) == 1
and 'same' not in self.A[state_a])):
same = ''
if 'same' in self.A[state_a]:
same = self.checkIfSameStateNotNeeded(
state_a, self.cleanerS(self.A[state_a]['same']))
# del(self.A[i]['same'])
for state_b in [
x.stateid for x in self._bfs_sort(
sorted(self.mma.states,
key=attrgetter('initial'),
reverse=True)[0])
]:
if state_b != state_a and state_b in self.A:
if state_a in self.A[state_b]:
temp = self.cleaner(self.A[state_b][state_a])
del (self.A[state_b][state_a])
for newTarget in self.A[state_a]:
if newTarget != 'same':
transValue = same + self.cleaner(
self.A[state_a][newTarget])
found = 1
add = temp + transValue
if newTarget == state_b:
newTarget = 'same'
add = '(' + self.replaceAlphabet(
add) + ')*'
if ((newTarget not in self.A[state_b]) or
(self.A[state_b][newTarget] == '')):
self.A[state_b][newTarget] = add
else:
self.A[state_b][
newTarget] = self.replaceAlphabetMixed(
self.A[state_b][newTarget],
add)
found = 1
if found == 1:
break
return found
def _solve(self):
self._create_map()
self._fix_pattern()
self.fixBrzozowskiBackwardLoopRemoval(
) # Remove backward paths because of kleene star
while 1:
found = self._Lexical()
if found == 0:
found = self.fixBrzozowskiAdvanced(found)
if found == 0:
break
if 'string' in self.A[0]:
return ''.join(self.A[0]['string'].splitlines())
def get_regex(self):
"""Generate regular expressions from DFA automaton"""
return self._solve()
def loadStatsLogToTextBox(k,lst,flag):
if len(statsPad.get('1.0', END))>0:
statsPad.delete('1.0', END)
try:
from ktail import kTails
import ktail
kt=kTails('K-TAILS')
kt.do_kTailEquivCheck(k, lst,flag)
ktfsm=kTailFSMGraph('FSM')
statsPad.insert(END,'Identifying Equivalent k-tails (k=' + str(box.get()) +')\n')
statsPad.insert(END,'-----------------------------------------------------------------------------------------------------------------------------------\n')
if len(kt.strEquiv)==0:
statsPad.insert(END, str('No Equivalent k-tails found with k=' + str(box.get())) +'\n')
if multitraceOption.get()==1:
if len(kt.strEquiv2)==0:
statsPad.insert(END, str('No Equivalent k-tails found with k=' + str(box.get())) +'for Trace 2\n')
for streq in kt.strEquiv:
statsPad.insert(END, str(streq)) #+ '\n')
statsPad.insert(END,'-----------------------------------------------------------------------------------------------------------------------------------\n')
statsPad.insert(END, 'Initial States in Trace 1: ' +str(kt.state)+ '\n')
statsPad.insert(END, 'Equivalent States in Trace 1: ' +str(kt.nodelist)+ '\n')
statsPad.insert(END, 'Merged States in Trace 1:' + str(kt.mergedlist)+ '\n')
statsPad.insert(END,'Mapping in Trace 1: ' +str(kt.mapping) +'\n')
statsPad.insert(END,'State Map Dictionary in Trace 1: '+str(ktfsm.stateMap) +'\n')
statsPad.insert(END,'Finalised States in Trace 1: '+ str(kt.getUniqueStates1)+'\n')
statsPad.insert(END,'++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n')
statsPad.insert(END,'State-Label in Trace 1: '+ str(kTailFSMGraph.transDict)+'\n')
statsPad.insert(END,'State Transitions:-----------------------------------------------------------------------------------------------------------------\n')
symbol=set()
for nx,kvx in ktfsm.stateMap.items():
for c in kvx:
statsPad.insert(END,str(nx) + '-->'+str(c) + '[label='+kvx[c] +']\n')
symbol.add(kvx[c])
#for ndfa in ktfsm.samplendfaloginfor:
# statsPad.insert(END,ndfa+'\n')
for ndfa in ktfsm.ndfaloginfor:
statsPad.insert(END,ndfa+'\n')
#reset the log lists
#ktfsm.samplendfaloginfor[:]
statsPad.insert(END,'***********************************************************************************************************************************\n')
if validateAgainstsample.get()==1:
statsPad.insert(END,'Processing Automata for Sample Input\n')
from dfa import DFA
if srcStateTextVariable is not None:
start_state=int(srcStateTextVariable.get())
print 'Initial State: ' + str(srcStateTextVariable.get())
statsPad.insert(END,'Initial State: '+ str(srcStateTextVariable.get())+'\n')
else:
tkMessageBox.showerror("Initial State","Not Intial state defined. Please check the stting panel")
#return
accept_states=set()
if len(acceptStatestEntry.get())>0:
for item in str(acceptStatestEntry.get()).split(','):
if item.isdigit():
accept_states.add(int(item))
else:
acceptrejectStatusvalue['text']=''
acceptrejectStatusvalue['bg']='grey'
tkMessageBox.showerror("State Error",'Accepting state requires an integer value')
return
print 'Accepting States: ' + str(acceptStatestEntry.get())
statsPad.insert(END,'Accepting State: ' + str(acceptStatestEntry.get())+'\n')
else:
tkMessageBox.showerror("Accepting state","No accepting states defined. Please check the setting panel")
getUniqueStatesSample=ktail.getUniqueStatesSample
print 'Sample States: ' + str(getUniqueStatesSample)
statsPad.insert(END,'Sample States: ' + str(getUniqueStatesSample)+'\n')
sampleTransitionmapping=ktail.sampleTransitionmapping
print 'Sample Transitions: ' + str(sampleTransitionmapping)
statsPad.insert(END,'Sample Transitions:\n')
for k,v in sampleTransitionmapping.items():
p,q=k
statsPad.insert(END,str(p)+'--->'+str(v)+'[label='+q+']\n')
#alphabet=[]
#for c in ktail.samplealphabet:
# alphabet.append(c)
print 'Alphabet from sample: ' + str( ktail.samplealphabet)
statsPad.insert(END,'Alphabet from sample: ' + str(ktail.samplealphabet)+'\n')
d=DFA(getUniqueStatesSample, ktail.samplealphabet,sampleTransitionmapping,start_state,accept_states,1);
alphabetfromtrace=kTailFSMGraph.alphabetfromtrace
if len(ktail.samplealphabet)==0:
tkMessageBox.showwarning("Process Sample", "Please proccess the sample automata first")
return
testalphabet=[]
for k,v in alphabetfromtrace.items():
p,q=k
testalphabet.append(q)
print 'Input Alphabet from Trace ' + str(testalphabet)
statsPad.insert(END,'Input Alphabet from Trace ' + str(testalphabet)+'\n')
sampleStatus=''
if d.run_with_input_list(testalphabet):
sampleStatus='AUTOMATA ACCEPTED'
acceptrejectStatusvalue['text']=sampleStatus
acceptrejectStatusvalue['bg']='green'
print 'Status:'+sampleStatus
else:
sampleStatus='AUTOMATA REJECTED'
acceptrejectStatusvalue['text']=sampleStatus
acceptrejectStatusvalue['bg']='red'
print 'ACCEPT/REJECT Status:'+sampleStatus
statsPad.insert(END,'++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n')
for l in d.loginformation:
print l
statsPad.insert(END,l+'\n')
d.loginformation=[] #reset the log information dictionary
statsPad.insert(END,'Status: ' + sampleStatus+'\n')
statsPad.insert(END,'++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n')
statsPad.configure(state='disabled')
except (ValueError):
tkMessageBox.ERROR
class DFATest(unittest.TestCase):
def test_simple_language(self):
# Language = {a^nb : n >= 0}
m = {
"states": ["q0", "q1", "q2"],
"alphabet": ["a", "b"],
"initial": "q0",
"terminals": ["q1"],
"delta": {
("q0", "a"): "q0",
("q0", "b"): "q1",
("q1", "a"): "q2",
("q1", "b"): "q2",
("q2", "a"): "q2",
("q2", "b"): "q2",
},
}
self.my_dfa = DFA()
self.my_dfa.load(m)
self.assertTrue(self.my_dfa._accepts("ab"))
self.assertTrue(self.my_dfa._accepts("aaaab"))
self.assertTrue(self.my_dfa._accepts("b"))
self.assertFalse(self.my_dfa._accepts("aba"))
self.assertFalse(self.my_dfa._accepts("aabbabaa"))
self.assertFalse(self.my_dfa._accepts("aaaaa"))
def test_load_from_yaml(self):
self.my_dfa = DFA()
self.my_dfa.load_from_yaml("example.yaml")
self.assertTrue(self.my_dfa._accepts("ab"))
self.assertTrue(self.my_dfa._accepts("aaaab"))
self.assertTrue(self.my_dfa._accepts("b"))
self.assertFalse(self.my_dfa._accepts("aba"))
self.assertFalse(self.my_dfa._accepts("aabbabaa"))
self.assertFalse(self.my_dfa._accepts("aaaaa"))
from dfa import DFA
dfa = DFA()
key = "2B7E151628AED2A6ABF7158809CF4F3C"
message = "3243F6A8885A308D313198A2E0370734"
fault = [0x1e, 0xe1, 0xb3, 0x16, 0x9e]
dfa.encrypt(key, message, 9, 0, fault)
dfa.reset()
# exploit faults
exploit_list = [
["de25841d02dc0962dc11c297193b0b32", "3925841d02dc09fbdc118597196a0b32"],
["f325841d02dc097edc11719719510b32", "3925841d02dc09fbdc118597196a0b32"],
["4025841d02dc09f9dc118f97196e0b32", "3925841d02dc09fbdc118597196a0b32"],
["1625841d02dc09bfdc11659719d20b32", "3925841d02dc09fbdc118597196a0b32"],
["9b25841d02dc09d5dc115c97197e0b32", "3925841d02dc09fbdc118597196a0b32"]
]
dfa.exploit(9, 0, exploit_list)
def main():
dfa1 = DFA() #This is the initial DFA defined by the exampleDFA.txt
dfa2 = DFA(
) #If this is the intersection it houses the second DFA if there is no second DFA it will go unused
testString = ""
'''
Opening the files and prompting the user if there are any errors.
'''
try:
dfa_filename = sys.argv[
1] #First command line argument that will be the file containing the DFA
dfa_file = open(dfa_filename, 'r')
if len(sys.argv
) != 2: #If there is no second argument given in the command
secondFilename = sys.argv[
2] #Second command line argument that will be the filecontaining the test cases
secondFile = open(secondFilename, 'r')
dfa2.makeDfa(
secondFile
) #Creating the dfa from the second file using the dfa class
except IndexError:
#This error occurs when one of the command line arguments are missing
sys.exit(
"The program needs two arguments to run.\nExample: dfaSim.py dfa.txt tests.txt"
)
except FileNotFoundError as err:
#This error occurs if any of the files do not exist
sys.exit("Unable to locate file: {0}".format(str(err).split()[-1]))
dfa1.makeDfa(dfa_file) #Creating the dfa from the file using the dfa class
if dfa2.transitionStates == []:
dfa1.acceptingStates = comp(dfa1)
dfa1.printDfa()
newTmpTransitionTable = []
'''
for i in range(len(dfa1.transitionStates)):
for j in range(len(dfa1.alphabet) - 1):
tmpArray = []
for k in range(len(dfa2.alphabet) - 1):
tmpArray2 = []
tmpArray2.append(int(dfa1.transitionStates[i][j]))
tmpArray2.append(int(dfa2.transitionStates[i][k]))
tmpArray.append(tmpArray2)
newTmpTransitionTable.append(tmpArray)
'''
'''
for l in range(len(dfa2.transitionStates)):
tmpArray2 = []
tmpArray2.append(int(dfa1.transitionStates[i][j]))
tmpArray2.append(int(dfa2.transitionStates[k][l]))
tmpArray.append(tmpArray2)
newTmpTransitionTable.append(tmpArray)
'''
'''
for k in range(len(dfa1.alphabet) - 1):
tmpArray2 = []
tmpArray2.append(int(dfa1.transitionStates[i][j]))
tmpArray2.append(int(dfa2.transitionStates[i][k]))
tmpArray.append(tmpArray2)
newTmpTransitionTable.append(tmpArray)
'''
newTransitionTable = []
transitions = []
for i in range(len(dfa1.transitionStates)):
for j in range(len(dfa1.transitionStates)):
transitions.append([i, j])
for i in range(len(newTmpTransitionTable)):
tmp = []
for j in range(len(newTmpTransitionTable[i])):
tmp.append(transitions.index(newTmpTransitionTable[i][j]))
newTransitionTable.append(tmp)
for i in newTransitionTable:
counter = 0
for j in i:
if counter == len(i) - 1:
print(j, end="")
else:
print(j, end=" ")
counter = counter + 1
print()
print(transitions)
print(len(transitions))
def test_prob_pred():
assert prob_pred(lift(universal({0, 1})), pred=bool, horizon=3) == 1
assert prob_pred(lift(empty({0, 1})), pred=bool, horizon=3) == 0
or_pdfa = lift(DFA(start=0, label=bool, inputs={0, 1}, transition=max))
assert prob_pred(or_pdfa, pred=bool, horizon=3) == pytest.approx(2**-3)
def CalculateTotalXC(self, params_up = None, params_down = None):
if params_up is None and params_down is None:
params_up = self.params_up
params_down = self.params_down
rho_up_arr = params_up[0]
rho_down_arr = params_down[0]
rho_arr = rho_up_arr + rho_down_arr
cfx_eps_xc = np.zeros(self.kskernel.ngrid)
cfx_xc = 0.0
# 1 A,
# 2 LSD
# 2.1 LSD x
# 2.2 LSD xc
# 3 PBE
# 3.1 PBE x
# 3.2 PBE xc
# 4 eps_x_exact
# for now, we store this after we've processed kskernel.
exks = ExKS(self.mol, self.kskernel, 'exks,')
self.kskernel.exact_eps_x_up, self.kskernel.exact_eps_x_down = exks.CalculateEpsilonX()
lsd = DFA(self.mol, self.kskernel, 'LDA,PW_MOD')
lsd_eps_x_up_arr, lsd_eps_x_down_arr, lsd_eps_c_arr = lsd.CalculateEpsilonXC()
lsd_eps_x_arr = lsd_eps_x_up_arr + lsd_eps_x_down_arr
pbe = DFA(self.mol, self.kskernel, 'PBE,PBE')
pbe_eps_x_up_arr, pbe_eps_x_down_arr, pbe_eps_c_arr = pbe.CalculateEpsilonXC()
pbe_eps_x_arr = pbe_eps_x_up_arr + pbe_eps_x_down_arr
b03_xhole = B03(self.mol, self.kskernel)
brn_xhole = BRN(self.mol, self.kskernel)
for gridID in range(self.kskernel.ngrid):
weight = self.kskernel.weights[gridID]
rho_up = rho_up_arr[gridID]
rho_down = rho_down_arr[gridID]
rho = rho_arr[gridID]
rs = self.kskernel.rs[gridID]
kf = self.kskernel.kf[gridID]
kf_up = (3. * np.pi**2 * rho_up)**(1. / 3.)
kf_down = (3. * np.pi**2 * rho_down)**(1. / 3.)
zeta = self.kskernel.zeta[gridID]
Q_up = self.kskernel.Q_up[gridID]
Q_down = self.kskernel.Q_down[gridID]
# print('Parameters: ru = {:.12e}\trd = {:.12e}\tqu = {:.12e}\tqd = {:.12e}'.format(rho_up, rho_down, Q_up, Q_down))
lsd_eps_x_up = lsd_eps_x_up_arr[gridID]
lsd_eps_x_down = lsd_eps_x_down_arr[gridID]
lsd_eps_c = lsd_eps_c_arr[gridID]
lsd_eps_xc = (rho_up * lsd_eps_x_up + rho_down * lsd_eps_x_down)/rho + lsd_eps_c
pbe_eps_x_up = pbe_eps_x_up_arr[gridID]
pbe_eps_x_down = pbe_eps_x_down_arr[gridID]
pbe_eps_c = pbe_eps_c_arr[gridID]
pbe_eps_xc = (rho_up * pbe_eps_x_up + rho_down * pbe_eps_x_down)/rho + pbe_eps_c
exact_eps_x_up = self.kskernel.exact_eps_x_up[gridID]
exact_eps_x_down = self.kskernel.exact_eps_x_down[gridID]
if rho > 1.0e-8:
A = self.CalculateA(rs, zeta)
B = self.CalculateB(rs, zeta)
# LSD CF
# print('Parameters: r = {:.12e}\tkf = {:.12e}\txc_LSD = {:.12e}'.format(rho, kf, lsd_eps_xc))
LSD_JXBRN = brn_xhole.CalculateJX(rho_up, rho_down, lsd_eps_x_up, lsd_eps_x_down)
# print(LSD_JXBRN)
E = self.CalculateE(rho, kf, lsd_eps_xc, LSD_JXBRN, A, B)
# PBE CF
PBE_JXBRN = brn_xhole.CalculateJX(rho_up, rho_down, pbe_eps_x_up, pbe_eps_x_down)
C, D = self.CalculateCD(rho, kf, pbe_eps_xc, PBE_JXBRN, A, B, E)
# return 0.0
# CFX
EXACT_JXB03 = b03_xhole.CalculateJX(rho_up, rho_down, Q_up, Q_down, exact_eps_x_up, exact_eps_x_down)
#to calculate C and energy
m1 = self.CalculateMomentJXE(1, EXACT_JXB03, E)
m2 = self.CalculateMomentJXE(2, EXACT_JXB03, E)
m3 = self.CalculateMomentJXE(3, EXACT_JXB03, E)
m4 = self.CalculateMomentJXE(4, EXACT_JXB03, E)
m5 = self.CalculateMomentJXE(5, EXACT_JXB03, E)
m6 = self.CalculateMomentJXE(6, EXACT_JXB03, E)
C = -(3. * np.pi / 4. + m2 * A + m3 * B + m6 * D) / m4
cfx_eps_xc[gridID] = 2. * np.pi * rho / kf ** 2 *(A * m1 + B * m2 + C * m3 + D * m5)
# cfx_xc += weight * rho * cfx_eps_xc[gridID]
cfx_xc = np.sum(self.kskernel.weights * rho_arr * cfx_eps_xc)
return cfx_xc
from dfa import DFA
import pytest
import random
import time
ban_words_set = set()
ban_words_list = list()
example100k = list()
dfa = DFA()
def test_get_words():
with open('sensitive_words.txt', 'r', encoding='utf-8-sig') as f:
for s in f:
if s.find('\\r'):
s = s.replace('\r', '')
s = s.replace('\n', '')
s = s.strip()
if len(s) == 0:
continue
if str(s) and s not in ban_words_set:
ban_words_set.add(s)
ban_words_list.append(str(s))
for _ in range(10000):
example100k.append(str_generator())
def str_generator():
str_test = str()
for _ in range(random.randint(1, 200)):
if random.random() < 0.1:
str_test += random.choice(ban_words_list)
class Converter:
def __init__(self, nfa, alpha):
self.nfa = nfa
self.alpha = alpha
# returns all states reachable through s by taking char c
# is c is None, does one level of spontaneaous transitions
def edge(self, s, c):
res = set()
for d in range(self.nfa.sc):
if c in self.nfa.edges(s, d):
res.add(d)
return res
# S set of states
# returns all set reachables from any sate in S, using only spontaneaous transitions
def closure(self, S):
S = set(S)
while True:
copy = set(S)
for s in copy:
for x in self.edge(s, None):
S.add(x)
if S == copy:
break
return S
# let d a set of states in NFA (equivalent to one state in the DFA)
# returns a new states of states in NFA (equivalent to one state in the DFA)
# rechable from tacking char c from any state in d, then taking spontaneaous transitions
def dfa_edge(self, d, c):
succ = set()
for s in d:
for x in self.edge(s, c):
succ.add(x)
return self.closure(succ)
def find_state(self, st):
for i, other in enumerate(self.states):
if other == st:
return i
return None
def build(self):
self.states = []
self.doc = Program.instance().add_doc(MdGfmDoc, "NFA to DFA")
self.doc.h1("DFA construction")
# First state is empty
# This is the error state
# All invalid transitions are stuck in this state
self.states.append(set())
self.dump_state(0)
# Second state is the eps-closure of the start state
self.states.append(self.closure({self.nfa.start}))
self.dump_state(1)
self.trans_list = []
j = 0
while j < len(self.states):
for c in self.alpha.larr:
# compute new state from state j, taking char c
e = self.dfa_edge(self.states[j], c)
# check if same set of states already exists
i = self.find_state(e)
if i is None:
# add new state
i = len(self.states)
self.states.append(e)
self.dump_state(i)
# record transition
self.trans_list.append((j, c, i))
j += 1
# Compute list of final states
# one state is final if any of it's NFA states is final
self.finals = [
d for (d, st) in enumerate(self.states) if self.nfa.final in st
]
self.dump_finals()
# Build DFA
self.dfa = DFA(len(self.states),
start_state=1,
final_states=self.finals,
err_state=0)
for t in self.trans_list:
self.dfa.add_transition(s0=t[0], s1=t[2], c=t[1])
self.doc.h1('DFA')
self.doc.write(self.dfa)
return self.dfa
def dump_state(self, idx):
st = self.states[idx]
self.doc.write('- Inserted state {}: `{{'.format(idx))
for s in st:
self.doc.write('{}; '.format(s))
self.doc.write('}`\n')
def dump_finals(self):
self.doc.write('- Final states: `{')
for s in self.finals:
self.doc.write('{}; '.format(s))
self.doc.write('}`\n')
def toDFA(self):
powerset = []
for i in range(1, (1 << len(self._states))):
sset = set()
for j in range(len(self._states)):
if i & (1 << j) != 0:
sset.add(self._states[j])
powerset.append(sset)
powerset = list(map(self._eclosure, powerset))
#print(len(powerset))
#print(powerset)
alph = self._alphabet
alph.remove('')
dfa = DFA(alph)
for state in powerset[0]:
if state.isFinal():
dfa.getState(0).setFinish(True)
for i in range(1, len(powerset)):
dfa.addState(State())
for state in powerset[i]:
if state.isFinal():
dfa.getState(i).setFinish(True)
break
for i in range(len(powerset)):
for c in alph:
dstSet = set()
#print('setting transition for : ' + str((dfa.getState(i), c)))
for s in powerset[i]:
dstSet = dstSet.union(
self._eclosure(set(self._transitions[(s, c)])))
dfa.setTransition(dfa.getState(i), c,
dfa.getState(powerset.index(dstSet)))
dfa.trim()
return dfa
class SLRParser:
def __init__(self, fileName, cFile):
self.e = Error(fileName)
self.nodeStack = []
self.fileName = fileName
self.grammar = Grammar(open(fileName).read())
self.dfa = DFA(self.grammar)
self.stack = [("DummySymbol", self.dfa.startState)]
self.lexer = Lexer(cFile, cFile + ".lex")
"""
This bit's going to go away soon
self.lexed = self.lexer.lexed
self.lexed = [x for x in self.lexed.split("\n") if x != '']
self.lexed = [eval(x) for x in self.lexed]
"""
self.lexed = self.lexer.lexedList
self.action = []
self.terminals = self.grammar.terminals
self.actions = {}
# construct action table
for state in self.dfa.transitionTable:
self.actions[state] = {}
for on in self.dfa.transitionTable[state]:
self.actions[state][on] = ("error")
self.actions[state]["$"] = ("error")
for state in self.dfa.transitionTable:
if "DummyStart -> " + self.grammar.startSymbol + " " + itemSeparator in state:
if state not in self.actions:
self.actions[state] = {"$" : ("accept")}
else:
self.actions[state]["$"] = ("accept")
for transition in self.dfa.transitionTable[state]:
actionState = self.dfa.goto(state, transition)
if any([itemSeparator + " " + transition in x for x in state]) and actionState is not None:
if state not in self.actions:
self.actions[state] = {transition : ("shift", actionState)}
else:
self.actions[state][transition] = ("shift", actionState)
if any([x[-1] == itemSeparator for x in state]):
matches = [x for x in state if x[-1] == itemSeparator]
matches = [x for x in matches if transition in self.grammar.getFollowSet(x.partition(" ")[0])]
for match in matches:
if match.partition(" ")[0] != "DummyStart":
reduceNum = len([x for x in match.partition(" -> ")[2].split(" ") if x != itemSeparator])
if state not in self.actions:
self.actions[state] = {transition : ("reduce", match.partition(" ")[0], transition, reduceNum)}
else:
self.actions[state][transition] = ("reduce", match.partition(" ")[0], transition, reduceNum)
def parse(self):
lexIndex = 0
self.lexed.append((0,'KEY','$'))
while True:
lexItem = self.getLexItem(self.lexed[lexIndex])
action = self.actions[self.stack[-1][1]][lexItem]
if action[0] == "shift":
self.stack.append((self.getLexItem((self.lexed[lexIndex])), action[1], self.lexed[lexIndex]))
self.nodeStack.append(lexItem)
lexIndex += 1
elif action[0] == "reduce":
children = []
for pop in range(action[3]):
lexVal = self.nodeStack.pop()
if lexVal in ('ID', 'NUM', 'TYPE'):
lexVal = AST(lexVal, [self.stack[-1][2][2]], self.lexed[lexIndex][0])
children.append(lexVal)
self.stack.pop()
children = list(reversed(children))
self.stack.append((action[1], self.dfa.goto(self.stack[-1][1], action[1]), self.lexed[lexIndex]))
astChildren = [x for x in children if isinstance(x, AST)]
if astChildren:
self.nodeStack.append(AST(action[1], children, min([x.lineNum for x in astChildren])))
else:
self.nodeStack.append(AST(action[1], children, self.lexed[(lexIndex)][0]))
elif action == ("error"):
# Debugging like a pro
print( self.lexed[lexIndex])
print( )
print( "Syntax Error:")
errorLine = self.lexed[lexIndex][0]-1
print( "Found on line: " + str(errorLine))
for offset in range(-3, 3):
if (errorLine+offset) >= 0 and (errorLine+offset) < len(self.lexer.infileLines):
pref = ' ' if (offset != 0) else '>>'
print( pref + self.lexer.infileLines[errorLine + offset])
print( "==========")
print( lexIndex)
print( "lexItem = " + str(lexItem) + " -> " + str(self.lexed[lexIndex]))
print( "action = " + str(action))
print( "STACK")
print(("...."))
for x in self.stack[-4:-1]:
print( x[0])
print( self.stack[-1][1])
a = self.dfa.transitionTable[self.stack[-1][1]]
b = {}
for key in a:
if a[key] != () and self.grammar.isTerminal(key):
b[key] = a[key]
print( b)
return None
elif action == ("accept"):
break
return self.nodeStack
def getLexItem(self, lexItem):
if lexItem[1] in ["KEY","SYM"]:
return lexItem[2]
else:
return lexItem[1]
class StateRemoval:
"""Transforms a pyfst DFA to regular expressions"""
def __init__(self, input_fst_a, alphabet=None):
"""
Args:
input_fst_a (DFA): The DFA states
alphabet (list): the input alphabet
Returns:
None
"""
if alphabet is None:
alphabet = createalphabet()
self.mma = DFA(self.alphabet)
self.mma.init_from_acceptor(input_fst_a)
self.alphabet = alphabet
self.l_transitions = {}
self.epsilon = ''
self.empty = None
def star(self, input_string):
"""
Kleene star operation
Args:
input_string (str): The string that the kleene star will be made
Returns:
str: The applied Kleene star operation on the input string
"""
if input_string != self.epsilon and input_string != self.empty:
return "(" + input_string + ")*"
else:
return ""
def _state_removal_init(self):
"""State Removal Operation Initialization"""
# First, we remove all multi-edges:
for state_i in self.mma.states:
for state_j in self.mma.states:
if state_i.stateid == state_j.stateid:
self.l_transitions[
state_i.stateid, state_j.stateid] = self.epsilon
else:
self.l_transitions[
state_i.stateid, state_j.stateid] = self.empty
for arc in state_i.arcs:
if arc.nextstate == state_j.stateid:
if self.l_transitions[state_i.stateid, state_j.stateid] != self.empty:
self.l_transitions[state_i.stateid, state_j.stateid] \
+= self.mma.isyms.find(arc.ilabel)
else:
self.l_transitions[state_i.stateid, state_j.stateid] = \
self.mma.isyms.find(arc.ilabel)
def _state_removal_remove(self, k):
"""
State Removal Remove operation
l_transitions[i,i] += l_transitions[i,k] . star(l_transitions[k,k]) . l_transitions[k,i]
l_transitions[j,j] += l_transitions[j,k] . star(l_transitions[k,k]) . l_transitions[k,j]
l_transitions[i,j] += l_transitions[i,k] . star(l_transitions[k,k]) . l_transitions[k,j]
l_transitions[j,i] += l_transitions[j,k] . star(l_transitions[k,k]) . l_transitions[k,i]
Args:
k (int): The node that will be removed
Returns:
None
"""
for state_i in self.mma.states:
for state_j in self.mma.states:
if self.l_transitions[state_i.stateid, k] != self.empty:
l_ik = self.l_transitions[state_i.stateid, k]
else:
l_ik = ""
if self.l_transitions[state_j.stateid, k] != self.empty:
l_jk = self.l_transitions[state_j.stateid, k]
else:
l_jk = ""
if self.l_transitions[k, state_i.stateid] != self.empty:
l_ki = self.l_transitions[k, state_i.stateid]
else:
l_ki = ""
if self.l_transitions[k, state_j.stateid] != self.empty:
l_kj = self.l_transitions[k, state_j.stateid]
else:
l_kj = ""
if self.l_transitions[state_i.stateid, state_i.stateid] != self.empty:
self.l_transitions[state_i.stateid, state_i.stateid] += l_ik + \
self.star(self.l_transitions[k, k]) + l_ki
else:
self.l_transitions[state_i.stateid, state_i.stateid] = l_ik + \
self.star(self.l_transitions[k, k]) + l_ki
if self.l_transitions[state_j.stateid, state_j.stateid] != self.empty:
self.l_transitions[state_j.stateid, state_j.stateid] += l_jk + \
self.star(self.l_transitions[k, k]) + l_kj
else:
self.l_transitions[state_j.stateid, state_j.stateid] = l_jk + \
self.star(self.l_transitions[k, k]) + l_kj
if self.l_transitions[state_i.stateid, state_j.stateid] != self.empty:
self.l_transitions[state_i.stateid, state_j.stateid] += l_ik + \
self.star(self.l_transitions[k, k]) + l_kj
else:
self.l_transitions[state_i.stateid, state_j.stateid] = l_ik + \
self.star(self.l_transitions[k, k]) + l_kj
if self.l_transitions[state_j.stateid, state_i.stateid] != self.empty:
self.l_transitions[state_j.stateid, state_i.stateid] += l_jk + \
self.star(self.l_transitions[k, k]) + l_ki
else:
self.l_transitions[state_j.stateid, state_i.stateid] = l_jk + \
self.star(self.l_transitions[k, k]) + l_ki
def _state_removal_solve(self):
"""The State Removal Operation"""
initial = sorted(
self.mma.states,
key=attrgetter('initial'),
reverse=True)[0].stateid
for state_k in self.mma.states:
if state_k.final:
continue
if state_k.stateid == initial:
continue
self._state_removal_remove(state_k.stateid)
print self.l_transitions
return self.l_transitions
def get_regex(self):
"""Regular Expression Generation"""
self._state_removal_init()
self._state_removal_solve()
return self.l_transitions[0]
def test_2():
sensitive_path = 'sensitive/sensitive.txt'
dfa = DFA()
dfa.load_from_pickle('automata/mydata.json')
dfa.accepts('我爱北京天安门\n')
dfa.accepts('我喜欢做假证\n')
dfa.accepts('我喜欢做嗳\n')
with open(sensitive_path, 'r', encoding='UTF-8') as f:
lines = f.readlines()
result = []
lines = [i for i in lines if i != '\n']
#for i in lines:
#print(i, '*70')
#i = ('我喜欢' + i)*70
#print(dfa.get_sensitive_words(i))
start_time = time.time()
for i in lines:
i = ('我喜欢' + i) * 2
dfa.get_sensitive_words(i)
result.append(True)
during_time = time.time() - start_time
print(r'耗时:', during_time)
print(r'平均z耗时:', during_time / len(lines))
start_time = time.time()
print(i)
i = i * 200
a = dfa.get_sensitive_words(i)
during_time = time.time() - start_time
print(a)
print(r'耗时:', during_time)
start_time = time.time()
for i in lines:
i = ('我喜欢' + i) * 4
dfa.get_sensitive_words(i)
result.append(True)
during_time = time.time() - start_time
print(r'耗时:', during_time)
print(r'平均z耗时:', during_time / len(lines))
start_time = time.time()
for i in lines:
i = ('我喜欢' + i) * 8
dfa.get_sensitive_words(i)
result.append(True)
during_time = time.time() - start_time
print(r'耗时:', during_time)
print(r'平均z耗时:', during_time / len(lines))
start_time = time.time()
for i in lines:
i = ('我喜欢' + i) * 16
dfa.get_sensitive_words(i)
result.append(True)
during_time = time.time() - start_time
print(r'耗时:', during_time)
print(r'平均z耗时:', during_time / len(lines))
result = np.array(result)
print(r'正确个数:', len(result[result == True]))
print(r'错误个数:', len(result[result == False]))
print(r'正确率:', len(result[result == True]) / len(result))
#!/usr/bin/env python3
#-*- coding: utf-8 -*-
import os
from dfa import DFA, load_delta_from_file
fn, ext = __file__.split(".")
module = fn
states = {"type": "fix", "content": ["A", "B", "C", "D"]}
start_state = "A"
accept_states = {"type": "fix", "content": ["D"]}
alphabet = "binary"
delta = load_delta_from_file("224b")
dfa = DFA(module, states, start_state, accept_states, alphabet, delta)
with open(fn + ".erl", "w") as f:
print(dfa.gen_erl_statem(), file=f)
from dfa import DFA
m = DFA()
m.load_from_yaml("automata/example.yaml")
#accepted! =)
m.accepts("b")
m.accepts("ab")
m.accepts("aab")
#rejected, =(
m.accepts("ba")
m.accepts("aba")
m.accepts("bbba")
