class Processor:
def __init__(self, request):
self._R = Request(request)
self._P = self._R.extractPatterns()
self._fileName = self._R.extractFileName()
self._F = File(self._fileName)
self._matchFile, self._unmatchFile = self._F.getPostProcessingFileNames()
self._M = None
self._UM = None
self._umCount = 0
self._mCount = 0
self._fileData = []
# Consumer Method.
def startConsumer(self, Q):
try:
while True:
text = Q.get()
notMatch = True
# Go through each pattern to see if it matches the line.
# If the line matches a pattern, break out of the for loop as
# there is no need to search for another match
for p in self._P:
g = Grok(p._pattern)
ans = g.match(text)
if ans != None:
# Write into <filename>_match.txt if there is a pattern match
# TODO find a way around the encoding issues
notMatch = False
ptext = ' '.join([x.encode("latin-1") + ':'.encode("latin-1") + y.encode("latin-1") for x, y in ans.items()])
self._M.write(ptext.decode("latin-1") + ' '.decode("latin-1") + p.getTag().decode("latin-1") + '\n'.decode("latin-1"))
self._mCount = self._mCount + 1
break
# If there is no match write into <filename>_unmatch.txt
if notMatch == True:
self._UM.write(text)
self._umCount = self._umCount + 1
Q.task_done()
except:
raise ProcessingException
# Start the Producer thread to feed the lines into the Queue
def beginProcessing(self):
try:
Q = Queue()
self._M = io.open(self._matchFile, 'w+', encoding="latin-1")
self._UM = io.open(self._unmatchFile, 'w+', encoding="latin-1")
t = Thread(target=self.startConsumer, args=(Q,))
t.daemon = True
t.start()
with io.open(self._fileName, 'r', encoding="latin-1") as f:
for text in f:
Q.put(text)
Q.join()
self.closeAllFiles()
return self._matchFile, self._unmatchFile, self._mCount, self._umCount
except:
raise ProcessingException
# Close all files
def closeAllFiles(self):
try:
self._M.close()
self._UM.close()
except:
raise FileProcessingException
