class DataManager(object):
"""
Contructor which is responsible for the data operations
"""
def __init__(self, appConfig, fileMgr, regexMgr, lmLgr):
self.lmLogger = lmLgr
self.metaDB = MetaDB()
self.fileManager = fileMgr
self.regexManager = regexMgr
self.keywordDict = {}
def createCustomer(self):
self.metaDB = MetaDB()
def getMetaDB(self):
return self.metaDB
def addAnalysisToCustomer(self, customer):
self.lmLogger.debug("Adding customer to db.")
"""
findFileNameString function gets the filename of the log file that is currently parsed
file - the file that is parsed to build the metaDB
"""
def findFileNameString(self, file):
filename = os.path.splitext(
os.path.basename(file))[0].__str__().split('.')
fileString = filename[1] + " " + filename[0]
print(fileString)
return fileString
"""
ImportLogFiles function process the ilog files to build hte metaDB
analysisId - the analysisId that is uniquely created
logFilenames - the logfiles passed in the zip file
"""
def importLogFiles(self, analysisId, logFilenames):
if (os.name == 'posix'):
import resource
self.lmLogger.info(
"Building associations from the given log files... Check the ViPR Log Manager log (ViPRLogManager.log), "
"to see which input log file is being analysed. ")
start_time = time.time()
for logfile in logFilenames:
logfilename = os.path.splitext(logfile)
lfname = None
suffix = None
if (len(logfilename) > 0):
lfname = os.path.basename(os.path.normpath(logfilename[0]))
if (len(logfilename) >= 1):
suffix = logfilename[1]
# if(os.name == 'posix'):
# self.lmLogger.info("Memory usage (in MB): " + str(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000))
self.lmLogger.debug("Working on the log file: " + lfname + suffix)
# initialization
endLineNumber = startLineNumber = 0
date = ""
keywordsToWrite = []
negateAssociationFlagMRow = False
#fileString = self.findFileNameString(logfile)
for line in fileinput.input(logfile):
#get the keywords for the current line that is read
keywords, negateAssociationFlag = self.regexManager.getKeywordsForLine(
line)
dateFromLine = self.regexManager.getDateFromLine(line)
# if date is present then it is a new line
if (not dateFromLine == ""):
#if this is not a first line then this is a new line where the fileString is found. So go ahead and store the start and end line numbers
if (not fileinput.isfirstline()
and (len(keywordsToWrite) > 0) and date != ""):
"""
The -if- condition ensures that we have looked the earlier lines for new line with file string. If first line then we should continue until we find the next fileString
write start line and end line.
"""
metaRow = MetaRow(date, fileinput.filename(),
startLineNumber, endLineNumber,
negateAssociationFlagMRow,
list(set(keywordsToWrite)))
self.metaDB.InsertData(metaRow)
lastInd = self.metaDB.getLastIndexFromMetaDB()
for eachKeyword in keywordsToWrite:
self.keywordDict.setdefault(eachKeyword,
list()).append(lastInd)
#reset the keywordsToWrite for next block
keywordsToWrite = []
negateAssociationFlagMRow = False
#This is the line number of the new line
endLineNumber = startLineNumber = fileinput.filelineno()
#get the date here, not earlier since this is the new line
date = self.regexManager.getDateFromLine(line)
#reset the list of keywords for the new block
keywordsToWrite += keywords
# logical or on the negate flag
negateAssociationFlagMRow = negateAssociationFlagMRow or negateAssociationFlag
else:
#the line is continuation of previous line so keep reading file
endLineNumber = fileinput.filelineno()
#save the read keywords
keywordsToWrite += keywords
negateAssociationFlagMRow = negateAssociationFlagMRow or negateAssociationFlag
else:
#this is the last line
if (len(keywordsToWrite) > 0 and date != ""):
metaRow = MetaRow(date, fileinput.filename(),
startLineNumber, endLineNumber,
negateAssociationFlagMRow,
list(set(keywordsToWrite)))
self.metaDB.InsertData(metaRow)
lastInd = self.metaDB.getLastIndexFromMetaDB()
for eachKeyword in keywordsToWrite:
self.keywordDict.setdefault(eachKeyword,
list()).append(lastInd)
#reset the keywordsToWrite for next block
keywordsToWrite = []
negateAssociationFlagMRow = False
fileinput.close()
self.lmLogger.debug(
"--- time to parse and build associations is %s seconds ---" %
round(time.time() - start_time))
self.lmLogger.debug(
"Total number of metarow (input log file - lines containing keywords) "
+ str(len(self.metaDB.tagsList)))
self.lmLogger.debug("Total number of unique keywords " +
str(len(self.keywordDict)))
self.lmLogger.debug("keyword dict size (in KB): " +
str(sys.getsizeof(self.keywordDict) / 1024))
self.lmLogger.debug("self.metaDB.tagsList size (in KB): " +
str(sys.getsizeof(self.metaDB.tagsList) / 1024))
if (os.name == 'posix'):
self.lmLogger.debug(
"Memory usage after making the associations (in MB): " +
str(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000))
