def load_and_search():
with open("ac_trie", "r+b") as bf:
mm = mmap.mmap(bf.fileno(), 0)
ac_first = AC.from_buff(mm, copy=False) # it shares memory
ac_second = AC.from_buff(mm, copy=False) # it shares memory
ac_third = AC.from_buff(mm, copy=False) # it shares memory
ac_four = AC.from_buff(mm, copy=False) # it shares memory
print("Matches after loading shared buffer:")
for id, start, end in ac_first.match(string_to_search):
print(id, string_to_search[start:end])
def test_buff_ac(self):
ac = AC.build([u"aİ", u"aaİ", u"aai̇", u"aai̇bİ"], True)
ac.save("ac.bin")
with open("ac.bin", "rb") as fi:
bs = bytearray(fi.read())
self.assertEqual(len(bs), ac.buff_size())
bs2 = bytearray(ac.buff_size())
ac.to_buff(bs2)
self.assertEqual(bs2, bs)
self._check_ac_correct(AC.from_buff(bs2, copy=True))
self._check_ac_correct(AC.from_buff(bs2, copy=False))
def get_shared_memory_find_matches(processname, shared_memory_tag, ac_size):
shm = shared_memory.SharedMemory(shared_memory_tag)
AC_in_bytes = shm.buf[0:ac_size]
ac_in_process = AC.from_buff(AC_in_bytes, copy=False)
string_to_search = "asdpythonasdasdruby"
print("Executing search in {}".format(processname))
for id, start, end in ac_in_process.match(string_to_search):
print(id, string_to_search[start:end])
#time.sleep(100)
ac_in_process = None
AC_in_bytes.release() # MUST release memory beforing closing shm insance, otherwise error is raised.
shm.close()
# Export to file
ac = AC.build(ac_patterns)
ac.save("ac_trie")
print("Matches before saving:")
for id, start, end in ac.match(string_to_search):
print(id, string_to_search[start:end])
# Force garbage collector, to avoid possible copy to child processes.
string_to_search = None
ac = None
ac_patterns = None
# Load from file with mmap and share memory
with open("ac_trie", "r+b") as bf:
mm = mmap.mmap(bf.fileno(), 0)
ac_first = AC.from_buff(mm, copy=False)
ac_size = ac_first.buff_size()
print("Size of AC automaton:Total patterns: {}bytes:{}patterns".format(ac_size, total_patterns))
ac_first = None # Force garbage collector.
# Function used by the children processes.
@profile
def get_shared_memory_find_matches(processname, shared_memory_tag, ac_size):
shm = shared_memory.SharedMemory(shared_memory_tag)
AC_in_bytes = shm.buf[0:ac_size]
ac_in_process = AC.from_buff(AC_in_bytes, copy=False)
string_to_search = "asdpythonasdasdruby"
print("Executing search in {}".format(processname))
for id, start, end in ac_in_process.match(string_to_search):
print(id, string_to_search[start:end])
#time.sleep(100)