def __fileOpen(self, filename):
self.filename = filename
self.message = vsb.VSBIOMalloc(self.size)
if not os.path.isabs(filename):
filename = os.path.realpath(filename)
self.handle = vsb.ReadVSB(filename)
self.state = ReadStatus.eFileOpened
def __next__(self): """ get next message from the buffer. """ self.state, size = vsb.ReadNextMessage(self.handle, self.message, self.size) if self.state == vsb.eSuccess: return VSBMessage(self.message, vsb.GetEDP(self.message), size) elif self.state == vsb.eEndOfFile: vsb.ReadClose(self.handle) raise StopIteration() elif self.state == vsb.eError: raise ValueError(self.get_error_message()) elif self.state == vsb.eBufferToSmall: vsb.VSBIOFree(self.message) self.message = vsb.VSBIOMalloc(size) self.size = size return self.__next__()
def __iter__(self):
if self.state == ReadStatus.eInit:
raise ValueError('VSBReader has not been initialized')
elif self.state == ReadStatus.eEndOfFile:
self.__fileOpen(self.filename)
if self.state != ReadStatus.eFileOpened:
raise ValueError('VSB file has not been opened')
elif self.state == ReadStatus.eError:
vsb.ReadClose(self.handle)
self.__fileOpen(self.filename)
if self.state != ReadStatus.eFileOpened:
raise ValueError('VSB file has not been opened')
return self
def __init__(self, filename):
if not os.path.isabs(filename):
filename = os.path.realpath(filename)
self.handle = vsb.WriteVSB(filename)
def write_msg(self, vsbMessage):
''' writes vsb message to file.'''
vsb.WriteMessage(self.handle, vsbMessage.info, vsbMessage.sizeOfMsg)
def __del__(self):
vsb.WriteClose(self.handle)
int(
datetime.fromisoformat(
config["EndTimeAbsolute"]).timestamp() * 1e9))
if len(MsgIDsPerNetwork) > 0:
TimeFilterString = " AND ((MessageTime > " + FilteredStartTime + ") AND (MessageTime < " + FilteredEndTime + "))"
else:
TimeFilterString = "((MessageTime > " + FilteredStartTime + ") AND (MessageTime < " + FilteredEndTime + "))"
log.info("Generate filtered vsb file")
if len(NetworkAndMsgIDQueryString) > 0:
TempQueryString = "(" + NetworkAndMsgIDQueryString + ")"
else:
TempQueryString = ""
msg_File.NumberOfRecordsInFilteredVSB = vsb.WriteFilteredVsb(
msg_File.DB_FileName, msg_File.FilteredVSBFilename,
TempQueryString + TimeFilterString, None)
if msg_File.NumberOfRecordsInFilteredVSB > 0:
sys.stdout.write("VSB file was created! with " +
str(msg_File.NumberOfRecordsInFilteredVSB) +
" in it. \n")
log.info("Finished generating filtered vsb file")
else:
sys.stdout.write('VSB file has no records for output.\n')
log.info('VSB file has no records for output.')
conn.close()
if ( os.path.isfile(msg_File.DB_FileName) and (msg_File.FileCreatedByClass) \
and (config["DeleteTempDB2FileAfterExecution"] == "TRUE") ):
os.remove(msg_File.DB_FileName)
def __init__(self, inputFilePaths, ReportGenTimeStamp):
self.FilesList = []
self.FilesListSorted = []
logging.basicConfig(level=logging.INFO)
self.log = logging.getLogger(__name__)
self.handler = logging.FileHandler('IPA.log')
self.handler.setLevel(logging.INFO)
# create a logging format
self.formatter = logging.Formatter('%(asctime)s - %(name)s - %(message)s')
self.handler.setFormatter(self.formatter)
self.log.addHandler(self.handler)
for inputFilePath in inputFilePaths:
filenameWithoutPath = os.path.basename(inputFilePath["path"])
filename, fileExtension = os.path.splitext(inputFilePath["path"])
self.FilesList.append(msgFile())
self.FilesList[-1].DB_FileName = os.path.splitext(filenameWithoutPath)[0] + "_" + ReportGenTimeStamp + "_Filtrd.db2"
self.FilesList[-1].FileName = filenameWithoutPath
self.FilesList[-1].FilePath = filename
self.FilesList[-1].FileExtension = fileExtension
self.FilesList[-1].FileCreatedByClass = False
self.FilesList[-1].FileStartTime = ""
self.FilesList[-1].FileEndTime = ""
self.FilesList[-1].FileStartTimeRaw = 0
self.FilesList[-1].FileEndTimeRaw = 0
if (fileExtension == ".vsb"):
try:
# Open the message database
self.log.info("Create database for file " + inputFilePath["path"])
vsb.CreateDatabase(inputFilePath["path"], self.FilesList[-1].DB_FileName, None)
self.FilesList[-1].FileCreatedByClass = True
conn = sqlite3.connect(self.FilesList[-1].DB_FileName, timeout=10)
except ValueError as e:
print(str(e))
else:
self.FilesList[-1].DB_FileName = inputFilePath["path"]
conn = sqlite3.connect(inputFilePath["path"], timeout=10)
self.log.info("Get info from DB file")
conn.text_factory = lambda x: str(x, 'utf-8', 'ignore')
# first get the list of networks and their IDs
cursor = conn.cursor()
cursor.execute("SELECT Min(FirstTime), Max(LastTime) FROM Network")
row = cursor.fetchone()
if row is not None:
startTime = row[0]
self.FilesList[-1].FileStartTimeRaw = startTime
self.FilesList[-1].FileStartTime = (datetime.fromtimestamp(startTime / 1e9, timezone.utc).isoformat() + '\n')
endTime = row[1]
self.FilesList[-1].FileEndTimeRaw = endTime
self.FilesList[-1].FileEndTime = (datetime.fromtimestamp(endTime / 1e9, timezone.utc).isoformat() + '\n')
#now get the list of existing network Ids and Names
cursor.execute("Select DISTINCT Id, Name from Network ORDER by Id")
row = cursor.fetchall()
if row is not None:
for i in range(len(row)):
self.FilesList[-1].FileneNetworks.append(file_network())
self.FilesList[-1].FileneNetworks[-1].network_id = row[i][0]
self.FilesList[-1].FileneNetworks[-1].network_name = row[i][1]
# now get the list of MsgIDs for each network
# NumMessages is an alias, which can be used as a shortcut in the where
for i in range(len(self.FilesList[-1].FileneNetworks)):
cursor.execute("SELECT Id, COUNT(*) NumMessages FROM RawMessageData WHERE NetworkId = " + str(self.FilesList[-1].FileneNetworks[i].network_id) + " GROUP BY Id")
row = cursor.fetchall()
for j in range(len(row)):
self.FilesList[-1].FileneNetworks[i].network_msg_ids_dec.append(row[j][0])
self.FilesList[-1].FileneNetworks[i].network_msg_ids_hex.append(hex(row[j][0]))
self.FilesList[-1].FileneNetworks[i].network_msg_num_msgs.append(row[j][1])
self.FilesListSorted = sorted(self.FilesList, key=lambda x: x.FileStartTimeRaw)
def get_display_message(self):
'''
return Display messages if any
'''
return vsb.GetDisplayMessage(self.handle)
def get_error_message(self):
'''
return Error messages if any.
'''
return vsb.GetErrorMessage(self.handle)
def get_progress(self):
'''
returns the progress as a integer percentage
'''
return vsb.GetProgress(self.handle)
def __del__(self):
vsb.VSBIOFree(self.message)
if self.state != vsb.eEndOfFile:
vsb.ReadClose(self.handle)
def get_byte_from_data(self, index):
if index > self.sizeOfMsg:
raise ValueError("index greater then size")
return vsb.GetByteFromData(self.info, index)
def __init__(self, inputFilePaths, ReportGenTimeStamp):
self.FilesList = []
self.FilesListSorted = []
logging.basicConfig(level=logging.INFO)
self.log = logging.getLogger(__name__)
self.handler = logging.FileHandler('IPA.log')
self.handler.setLevel(logging.INFO)
# create a logging format
self.formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(message)s')
self.handler.setFormatter(self.formatter)
self.log.addHandler(self.handler)
if IPAInterfaceLibrary.is_running_on_wivi_server():
OutputFilePath = os.path.dirname(sys.argv[0])
else:
OutputFilePath = os.path.dirname(sys.argv[0])
for inputFilePath in inputFilePaths:
self.FilesList.append(msgFile())
self.FilesList[-1].InputFileName = os.path.basename(
inputFilePath["path"])
self.FilesList[-1].InputFilePath = os.path.dirname(
inputFilePath["path"])
filename, fileExtension = os.path.splitext(
str(self.FilesList[-1].InputFileName))
self.FilesList[-1].DB_FileName = os.path.join(
OutputFilePath,
filename + "_" + ReportGenTimeStamp + "_Filtrd.db2")
self.FilesList[-1].OutputFilePath = OutputFilePath
self.FilesList[-1].FileExtension = fileExtension
self.FilesList[-1].FileCreatedByClass = False
self.FilesList[-1].FileStartTime = ""
self.FilesList[-1].FileEndTime = ""
self.FilesList[-1].FileStartTimeRaw = 0
self.FilesList[-1].FileEndTimeRaw = 0
self.FilesList[-1].TempVSBExtracted = False
zip2FolderName = ""
self.log.info("InputFileName: " +
str(self.FilesList[-1].InputFileName))
#now compress output files
if (fileExtension == ".zip"):
with zipfile.ZipFile(
os.path.join(self.FilesList[-1].InputFilePath,
self.FilesList[-1].InputFileName),
'r') as zipObj:
# Get a list of all archived file names from the zip
listOfFileNames = zipObj.namelist()
# Iterate over the file names
self.log.info("First level zip filenames:")
for zipfileName in listOfFileNames:
self.log.info(zipfileName)
# skip directories
if not zipfileName:
continue
if zipfileName[len(zipfileName) -
4:len(zipfileName)] == ('.vsb'):
zipObj.extract(zipfileName,
self.FilesList[-1].OutputFilePath)
self.FilesList[-1].InputFileName = zipfileName
self.FilesList[-1].FileExtension = ".vsb"
self.FilesList[-1].InputFilePath = self.FilesList[
-1].OutputFilePath
self.FilesList[-1].TempVSBExtracted = True
self.log.info("Only one level of zip")
elif zipfileName[len(zipfileName) -
8:len(zipfileName)] == ('.vsb.zip'):
self.log.info(".vsb.zip file found")
zip2FolderName = os.path.join(
OutputFilePath.replace('\\', '/'),
'tempZipFolderName').replace('\\', '/')
zipObj.extract(zipfileName, zip2FolderName)
zip3FileName = os.path.join(
zip2FolderName,
zipfileName).replace('\\', '/')
with zipfile.ZipFile(zip3FileName) as zipObj2:
zipObj2.extract(zipObj2.namelist()[0],
OutputFilePath)
self.FilesList[
-1].InputFileName = zipObj2.namelist()[0]
self.log.info(
str(self.FilesList[-1].InputFileName))
filename2, fileExtension2 = os.path.splitext(
self.FilesList[-1].InputFileName)
self.FilesList[
-1].FileExtension = fileExtension2
self.FilesList[
-1].InputFilePath = self.FilesList[
-1].OutputFilePath
self.FilesList[-1].TempVSBExtracted = True
try:
shutil.rmtree(zip2FolderName)
self.log.info("Removed top level zip file")
except OSError as e:
self.log.info("error removing temp zip folder")
if (self.FilesList[-1].FileExtension == ".vsb"):
try:
# Open the message database
self.log.info("Create database for file " + str(
os.path.join(self.FilesList[-1].InputFilePath,
self.FilesList[-1].InputFileName)))
self.log.info("vsb filename: " + str(
os.path.join(self.FilesList[-1].InputFilePath,
self.FilesList[-1].InputFileName)))
vsb.CreateDatabase(
os.path.join(self.FilesList[-1].InputFilePath,
self.FilesList[-1].InputFileName),
self.FilesList[-1].DB_FileName, None)
if ((self.FilesList[-1].TempVSBExtracted == True) and
(os.path.isfile(
os.path.join(self.FilesList[-1].InputFilePath,
self.FilesList[-1].InputFileName)))):
os.remove(
os.path.join(self.FilesList[-1].InputFilePath,
self.FilesList[-1].InputFileName))
self.log.info("Removed temp input vsb file")
self.FilesList[-1].FileCreatedByClass = True
conn = sqlite3.connect(self.FilesList[-1].DB_FileName,
timeout=10)
except ValueError as e:
print(str(e))
else:
self.FilesList[-1].DB_FileName = os.path.join(
self.FilesList[-1].InputFilePath,
self.FilesList[-1].InputFileName)
conn = sqlite3.connect(os.path.join(
self.FilesList[-1].InputFilePath,
self.FilesList[-1].InputFileName),
timeout=10)
self.log.info("Get info from DB file")
conn.text_factory = lambda x: str(x, 'utf-8', 'ignore')
# first get the list of networks and their IDs
cursor = conn.cursor()
cursor.execute("SELECT Min(FirstTime), Max(LastTime) FROM Network")
row = cursor.fetchone()
if row is not None:
startTime = row[0]
self.FilesList[-1].FileStartTimeRaw = startTime
self.FilesList[-1].FileStartTime = (datetime.fromtimestamp(
startTime / 1e9, timezone.utc).isoformat() + '\n')
endTime = row[1]
self.FilesList[-1].FileEndTimeRaw = endTime
self.FilesList[-1].FileEndTime = (datetime.fromtimestamp(
endTime / 1e9, timezone.utc).isoformat() + '\n')
#now get the list of existing network Ids and Names
cursor.execute("Select DISTINCT Id, Name from Network ORDER by Id")
row = cursor.fetchall()
if row is not None:
for i in range(len(row)):
self.FilesList[-1].FileNetworks.append(file_network())
self.FilesList[-1].FileNetworks[-1].network_id = row[i][0]
self.FilesList[-1].FileNetworks[-1].network_name = row[i][
1]
# now get the list of MsgIDs for each network
# NumMessages is an alias, which can be used as a shortcut in the where
for i in range(len(self.FilesList[-1].FileNetworks)):
self.FilesList[-1].FileNetworks[i].numberOfArbIDs = 0
cursor.execute(
"SELECT Id, COUNT(*) NumMessages FROM RawMessageData WHERE NetworkId = "
+ str(self.FilesList[-1].FileNetworks[i].network_id) +
" GROUP BY Id")
row = cursor.fetchall()
for j in range(len(row)):
self.FilesList[-1].FileNetworks[
i].network_msg_ids_dec.append(row[j][0])
self.FilesList[-1].FileNetworks[
i].network_msg_ids_hex.append(hex(row[j][0]))
self.FilesList[-1].FileNetworks[
i].network_msg_num_msgs.append(row[j][1])
self.FilesList[-1].FileNetworks[
i].numberOfArbIDs = self.FilesList[-1].FileNetworks[
i].numberOfArbIDs + 1
# now loop through each network and each ArbID and determine the min and max sample period
for i in range(len(self.FilesList[-1].FileNetworks)):
for j in range(
self.FilesList[-1].FileNetworks[i].numberOfArbIDs):
if self.FilesList[-1].FileNetworks[i].network_msg_num_msgs[
j] > 1:
QueryString = "CREATE VIEW IF NOT EXISTS TempView AS SELECT * FROM RawMessageData WHERE NetworkId = " + str(
self.FilesList[-1].FileNetworks[i].network_id
) + " AND ID = " + str(
self.FilesList[-1].FileNetworks[i].
network_msg_ids_dec[j])
cursor.execute(QueryString)
QueryString = "SELECT MIN(DeltaVal), MAX(DeltaVal) FROM (SELECT (a.MessageTime - b.MessageTime) as DeltaVal FROM TempView AS a JOIN TempView AS b ON (b.MessageTime = (SELECT MAX(z.MessageTime) FROM TempView AS z WHERE z.MessageTime < a.MessageTime)))"
cursor.execute(QueryString)
row = cursor.fetchall()
self.FilesList[-1].FileNetworks[
i].network_msg_id_min_periods.append(row[0][0] *
1e-9)
self.FilesList[-1].FileNetworks[
i].network_msg_id_max_periods.append(row[0][1] *
1e-9)
cursor.execute("DROP VIEW IF EXISTS TempView")
else:
self.FilesList[-1].FileNetworks[
i].network_msg_id_min_periods.append(0)
self.FilesList[-1].FileNetworks[
i].network_msg_id_max_periods.append(0)
self.FilesListSorted = sorted(self.FilesList,
key=lambda x: x.FileStartTimeRaw)
def get_message_time(self, msg): ''' return the message seconds since Jan 1, 2007 ''' return vsb.GetMsgTime(msg.info)
