def WorkerProc(self, fnKillSignalled, userData):
lastUpdate = 0
interval = Configuration.get().GetConnectionUpdateInterval()
buffer = "<?xml version=\"1.0\" encoding=\"utf-8\"?>"
buffer = buffer + "<Oscar Type=\"ConnectionInformation\">"
buffer = buffer + "<Version>1.0</Version>"
buffer = buffer + "<OscarVersion>" + VersionMgr.ReadVer(
) + "</OscarVersion>"
buffer = buffer + "<ID>" + Configuration.get().GetID() + "</ID>"
buffer = buffer + "<Port>" + str(Configuration.get(
).GetDownstreamConnection().getPort()) + "</Port>"
buffer = buffer + "</Oscar>"
#<?xml version="1.0" encoding="utf-8"?>
#<Oscar Type="ConnectionInformation">
# <Version>1.0</Version>
# <ID>Foo</Foo>
# <Port>Port</Port>
#</Oscar>
while not fnKillSignalled(
): # run until signalled to end - call passed function to check for the signal
if lastUpdate < Time.GetCurrMS() - interval:
TargetManager.GetTargetManager().BroadcastDownstream(
buffer, True, None
) # send Connection Data to all downstream things (Oscars & Marvins)
lastUpdate = Time.GetCurrMS()
Configuration.get().RescanTargets()
else:
Sleep.Sleep(0.25)
TargetManager.GetTargetManager(
).CheckForRemovalOfDynamicMarvins()
def HandleIncomingGroupPacket(self, rawData, node, fromAddress):
from Helpers import Configuration
if Configuration.get().GetBITW_Active():
rawData = Configuration.get().HandleBITWBuffer(
rawData) # handle Bump in the wire
if 0 != TargetManager.GetTargetManager().BroadcastDownstream(
rawData, False, node, True): # send to all - towards a Marvin
Statistics.GetStatistics().OnPacketChainedDownstream(rawData)
# Break up packet and do bump in the wire.
# If BITW not active, maybe we don't need to do this could be a lot of processing for nothing....
objGroupPacket = MarvinGroupData.MarvinDataGroup(
"", "", "", 0, "1.0", True)
for packet in node.getElementsByTagName('Oscar'):
objMarvinPacket = self.CreateMarvinData(packet, rawData,
fromAddress)
if None == objMarvinPacket:
return
objGroupPacket.AddPacket(objMarvinPacket)
# objData = MarvinData.MarvinData(namespace,ID,value,0,version,True)
GuiMgr.OnDataPacketSentDownstream(objGroupPacket, "Chained")
Recorder.get().AddData(objGroupPacket)
def SetTitle(self,titleStr):
name = Configuration.get().GetID()
if None == name:
name =""
name = "[" + name + "]"
TitleString = "Oscar " + name +" - " + VersionMgr.ReadVer() + " [" + str(Configuration.get().GetUpstreamConnection()) +"]" + titleStr
self.pGui.SetTitle(TitleString)
def StartupWorkerProc(fnKillSignalled, userData):
downstreamServer = userData[0]
upstreamServer = userData[1]
Sleep.SleepMs(500)
downstreamServer.Start()
upstreamServer.DropPackets(True)
upstreamServer.Start()
Watchdog.ConnectionUpdateTimer()
Watchdog.WatchdogTimer()
conf = Configuration.get()
if None != conf.GetAutorunFilename():
GuiMgr.OnStopLiveData()
#GuiMgr.OnStopPlayback()
#GuiMgr.OnStopRecording(True) #drop all recorded packets
GuiMgr.OnSetPlaybackSpeed(Configuration.get().GetPlaybackSpeed())
ss = Configuration.get().GetAutorunLocations()
#GuiMgr.OnEnablePlayback()
GuiMgr.ReadFromFile(Configuration.get().GetAutorunFilename())
GuiMgr.OnStopPlayback()
Sleep.SleepMs(
100) # let gui worker threads catch up, so gui updates properly
GuiMgr.OnStartPlayback()
GuiMgr.OnSetRepeatMode(Configuration.get().GetAutoRunMode(), ss[0],
ss[1])
else:
upstreamServer.DropPackets(False)
if None != conf.GetAutorunTime() and conf.GetAutorunTime(
) > 0: # specified a --time, so let's hang out for that long
endTime = Time.GetCurrMS() + conf.GetAutorunTime() * 60 * 1000
Log.getLogger().info("Waiting for " + str(conf.GetAutorunTime()) +
" minutes before auto shutdown")
if conf.GetRecordFilename():
GuiMgr.OnStartRecording()
while not fnKillSignalled() and endTime > Time.GetCurrMS():
Sleep.SleepMs(250)
Log.getLogger().info("Shutting down after time period")
if conf.GetRecordFilename(
): # was a recording session, so quit after that time
GuiMgr.OnStopRecording()
GuiMgr.WriteToFile(conf.GetRecordFilename())
Log.getLogger().info("Saving Recorded data to file: " +
conf.GetRecordFilename())
GuiMgr.Quit()
def PerformCollection(self):
from Helpers import Configuration # circular import if I do at top of file
returnVal = None
collectedValue = self.Collect()
#Get collected time after collection, can't be sure each collection
#take same amount of time
elapsedTime = self.GetElapsedTimeSinceLast()
self.SetLastCollectionTime(self.GetTimeMS())
if collectedValue != Collector.ErrorValue and elapsedTime > 0:
#Have collected data, now normalize, check bounds and send
sendValue = self.__NormalizeData(collectedValue, elapsedTime)
if self._Bound_Action != BoundAction.Invalid:
sendValue = self.__BoundData(sendValue)
if None == sendValue and self.ReadyForConsumption():
sendValue = "HelenKeller" # outside bounds, and has run through whole process once
refresh = self._RefreshRequested
# Some checking for SendOnlyOnDelta, since this is UDP
# going to send it n times just to make sure it gets there
if collectedValue == self._LastValue:
if self._SendOnlyOnDelta:
if self._SentValueCount < Configuration.GetTimesToRepeatPacket(
):
refresh = True
else:
self._SentValueCount = 0
if collectedValue == self._LastValue and self._SendOnlyOnDelta and not refresh: # nothing changed, and only want on change
pass
elif not self._DoNotSend and sendValue != "HelenKeller": # HelenKeller means it is a mute collector, so don't send the actual data
if True == self._Normalize and self._LastValue == Collector.__InitialMagicValue:
pass # skip this piece of data - it is normalized, but we have no previous data point to normalize against. If we don't skip, big jump on 1st datapoint in widgets
else:
returnVal = self.__CreateSendBuffer(
sendValue, elapsedTime, sendValue != collectedValue)
self._SentValueCount += 1
if True == self._Normalize and self._LastValue == Collector.__InitialMagicValue:
pass # skip this piece of data - it is normalized, but we have no previous data point to normalize against. If we don't skip, big jump on 1st datapoint in widgets
else:
self._LastSentValue = sendValue # if if don't send, update this because operators could use this value
self._LastValue = collectedValue
if not self.ReadyForConsumption():
self.EnableForConsumption()
self.HandleAnyExtraStuff()
if True == refresh:
self._RefreshRequested = False
return returnVal
def getNamespacesFromBuffer(self, buffer):
from Helpers import Configuration
config = Configuration.get()
start = '<Namespace>'
end = '</Namespace>'
reg = "(?<=%s).*?(?=%s)" % (start, end)
try:
result = re.search(reg, buffer)
r = re.compile(reg)
NamespaceList = r.findall(buffer)
if None == NamespaceList:
Log.getLogger().error(
"Error Something bad in trying to find namespaces HandleBITW "
+ buffer)
return None
list = []
map = config.GetBITW_NamespaceMap()
checkedMap = {}
for namespace in NamespaceList:
if not namespace in checkedMap:
checkedMap[
namespace] = namespace # could be a group, and only need to check once, but could be case differences, so don't use upper
strUp = namespace.upper()
if strUp in map:
list.append((namespace, map[strUp]))
except Exception as ex:
print(str(ex))
return list
def getLogger():
from Helpers import Configuration
if None == Logger._logger:
Logger._logger = logging.getLogger("Oscar")
logging.basicConfig(level=logging.ERROR,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename=Configuration.get().GetLogFilename(),
filemode='w')
console = logging.StreamHandler()
console.setLevel(logging.INFO)
# create formatter
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
# add formatter to console
console.setFormatter(formatter)
# add ch to logger
if Logger.LogToConsole == True:
Logger._logger.addHandler(console)
Logger._console = console
#Logger._errorFn = Logger._logger.error
#Logger._logger.error = Logger.myFn
return Logger._logger
def _BroadcastDownstream(self,
sendBuffer,
ignoreTimeout,
domNode,
isGroup=False):
from Helpers import Configuration
sentCount = 0
for targetKey in self._DownstreamTargets.keys():
if True == self.SendToDownstreamTarget(sendBuffer, targetKey,
ignoreTimeout):
sentCount += 1
if sentCount > 0:
Statistics.GetStatistics().OnPacketBroadcastDownstream()
if Configuration.get().GetShunting() and (None != domNode
or True == isGroup):
if None != domNode and not isGroup:
self.HandleShuntingData(domNode)
else: # is a group and need to go through each item individually
if None == domNode:
try:
dom = xml.dom.minidom.parseString(sendBuffer)
domNode = dom._get_firstChild()
except Exception as ex:
Log.getLogger().error(
"Error Something bad in Trying to read xml in BroadcastDownstream"
)
if None != domNode:
for dataNode in domNode.getElementsByTagName('Oscar'):
self.HandleShuntingData(dataNode)
return sentCount
def workerProc(self, fnKillSignalled, userData):
dataHandler = DataHandler.GetDataHandler()
from Helpers import Configuration
buffSize = Configuration.get().GetRecvBufferSize()
try:
while not fnKillSignalled(
): # run until signalled to end - call passed function to check for the signal
try:
data, fromAddress = self.m_socket.recvfrom(buffSize)
data = data.strip().decode("utf-8")
self.m_rxPackets += 1
self.m_rxBytes += len(data)
except: # socket.error:
#time.sleep(.01) #no data, so sleep for 10ms
continue
if not fnKillSignalled():
if False == self.__DropPackets():
dataHandler.HandleLiveData(data, fromAddress)
elif "<Marvin Type=\"Bullhorn\">" in data:
dataHandler.HandleLiveData(
data,
fromAddress) # drop all but the Marvin Bullhorn
except Exception as ex:
Log.getLogger().debug("Thread Error: " + str(ex) + " --> " +
traceback.format_exc())
def HandleIncomingOscarDatapacket(self, node, rawData, fromAddress):
from Helpers import Configuration
# Use this object to update the gui and for recording
# is also where BITW check update of checked map occurs
objData = self.CreateMarvinData(node, rawData, fromAddress)
if Configuration.get().GetBITW_Active():
rawData = Configuration.get().HandleBITWBuffer(
rawData) # handle Bump in the wire
if 0 != TargetManager.GetTargetManager().BroadcastDownstream(
rawData, False, node): # send to all - towards a Marvin
Statistics.GetStatistics().OnPacketChainedDownstream(rawData)
if None != objData:
GuiMgr.OnDataPacketSentDownstream(objData, "Chained")
Recorder.get().AddData(objData)
def recvTCP_WorkerProc_Client(self, fnKillSignalled, _):
from Helpers import Configuration
buffSize = Configuration.get().GetRecvBufferSize()
self._connect()
currPacket = None
try:
while not fnKillSignalled(
): # run until signalled to end - call passed function to check for the signal
if None == self.m_socket:
time.sleep(.5)
self._connect()
continue
try:
rawData = self.m_socket.recv(buffSize).decode("utf-8")
if len(rawData) == 0: # other end disconnected
time.sleep(.5)
self.m_socket.close()
self._connect()
currPacket = None
continue
self.m_rxBytes += len(rawData)
for dataByte in rawData:
if dataByte == TCP_PACKET_DELIMITER_START:
if None != currPacket:
# print("Received Start of Packet before an End Of Packet Delimeter")
pass
currPacket = ""
elif dataByte == TCP_PACKET_DELIMITER_END:
self.m_rxPackets += 1
self._processIncomingData(currPacket, None)
currPacket = None
else:
currPacket += dataByte
except socket.timeout:
pass
except Exception as _: # socket.error:
self.m_socket.close()
self.m_socket = None
pass
continue
if not fnKillSignalled():
pass
#dataHandler.HandleLiveData(rawData,clientAddr)
except Exception as ex:
Log.getLogger().debug("Thread Error: " + str(ex) + " --> " +
traceback.format_exc())
def HandleBullhornAnnouncement(self,node,rawData,fromAddr):
#<?xml version="1.0" encoding="utf-8"?>
#<Marvin Type="Bullhorn">
# <Version>1.0</Version>
# <UniqueID>3236</UniqueID>
# <Hostname>pgkutch.beervana.net</Hostname>
# <Key>md5 hash</Key>
# <Port>5000</Port>
#</Marvin>
try:
version = node.getElementsByTagName('Version')[0].firstChild.nodeValue
Hash = node.getElementsByTagName('Key')[0].firstChild.nodeValue
Port = node.getElementsByTagName('Port')[0].firstChild.nodeValue
UniqueID = node.getElementsByTagName('UniqueID')[0].firstChild.nodeValue
IP = fromAddr[0].lower()
Hostname = node.getElementsByTagName('Hostname')[0].firstChild.nodeValue
except Exception as _:
Statistics.GetStatistics().OnMalformedPacketReceived("Received invalid Marvin Bullhorn Packet : " + rawData)
return
RemoteKey = Configuration.get().GetMarvinAutoConnectKeyFromHash(Hash)
strID = Hostname + ":[" + IP + ":" + Port +"]"
if None == RemoteKey: #don't have anything configured that matches
Log.getLogger().warning("Received Marvin Dynamic Connection Message, with no corropsonding Key from: " + strID)
return
strID += " Key=" + RemoteKey
HashMapKey = IP + ":" + str(Port)
objExisting = TargetManager.GetTargetManager().GetDownstreamTarget(HashMapKey)
if None == objExisting: # could be NDS name not resolved, so try by IP address
objExisting = TargetManager.GetTargetManager().GetDownstreamTargetEx(IP,Port)
if None != objExisting:
if hasattr(objExisting,'_ReceivedOnUniqueID') and UniqueID != objExisting._ReceivedOnUniqueID:
Log.getLogger().warning("Received Marvin Dynamic Connection Message, for already active connection: " + strID)
else:
pass # is simply the additional packets (marvin sends multiples as it is UDP traffic)
return
# doesn't already exist, so let's to add!
if "1.0" == version:
objTarget = Target.Target(IP,Port,ConnectionType.DynamicMarvin,True)
else:
objTarget = Target.Target(IP,Port,ConnectionType.DynamicOscar,True)
objTarget._ReceivedOnUniqueID = UniqueID # so we can filter out dups due to UDP
objTarget._UserKey = RemoteKey
TargetManager.GetTargetManager().AddDownstreamTarget(objTarget,HashMapKey)
Log.getLogger().info("Creating Dynamic Connection:" + strID)
return
def ShuntWorkerProc(self, fnKillSignalled, userData):
from Helpers import Configuration
sleepTime = Configuration.get().GetShuntWorkerInterval()
try:
while not fnKillSignalled(
): # run until signalled to end - call passed function to check for the signal
self.__ShuntLock.acquire()
DupMap = None
try:
DupMap = self.__ShuntedDataByFile
self.__ShuntedDataByFile = {}
except Exception as Ex:
Log.getLogger().info(
"Unknown error in Shunt Worker Proc: " + str(Ex))
finally:
self.__ShuntLock.release()
if None != DupMap:
for file in DupMap.keys():
tfh, tfh_path = mkstemp(
) # create temp file, copy target file to it then do all updates and copy it
# back to original
close(tfh)
if os.path.exists(file):
copy(file, tfh_path) #copy contents to temp file
mapEntry = DupMap[file]
for shuntEntry in mapEntry:
namespace = shuntEntry[0]
ID = shuntEntry[1]
History = shuntEntry[2]
Value = shuntEntry[3]
if True == History:
self.ShuntHistory(namespace, ID, Value,
tfh_path)
else:
self.ShuntDynamicCollectorStyle(
namespace, ID, Value, tfh_path)
#all done processing this file, so copy temp file to
#real file
#Remove original file
try:
remove(file)
except Exception as Ex:
pass
#Move new file
move(tfh_path, file)
Sleep.SleepMs(sleepTime)
except Exception as Ex:
Log.getLogger().info("Unknown error in Shunt Worker Proc: " +
str(Ex))
def SetOverrideNamespaceString(self,newNamespaceString):
from Helpers import Configuration
self._NamespaceOverride = newNamespaceString
try:
for ns in Configuration.GetNamespaces():
if ns.GetID().lower() == newNamespaceString.lower():
Log.getLogger().warning("Setting OverrideNamespace for collector {0} to {1}. However that Namespace already exists - conflicts may occur.".format(newNamespaceString,self.GetID()))
break
except:
pass
def WatchdogProc(self, fnKillSignalled, userData):
lastUpdate = 0
interval = Configuration.get().GetTimeoutPeriod(
) * 0.25 # send a watchdog at 4x rate of timeout
buffer = "<?xml version=\"1.0\" encoding=\"utf-8\"?>"
buffer = buffer + "<Oscar Type=\"WatchdogTimer\">"
buffer = buffer + "<Version>1.0</Version>"
buffer = buffer + "<Port>" + str(
Configuration.get().GetUpstreamConnection().getPort()) + "</Port>"
buffer = buffer + "</Oscar>"
while not fnKillSignalled(
): # run until signalled to end - call passed function to check for the signal
if lastUpdate < Time.GetCurrMS() - interval:
TargetManager.GetTargetManager().BroadcastUpstreamToType(
buffer, ConnectionType.UpstreamOscar
) # send heartbeat to all upstream Oscars
lastUpdate = Time.GetCurrMS()
Sleep.Sleep(0.25) #snooze for 250 ms
def __initialize(self):
self._UpstreamTargets = {} # data being sent towards Minion
self._DownstreamTargets = {} # data being sent torwads Marvin
TargetManager._instance = self
from Helpers import Configuration
self.__UseGoverner = Configuration.get().GetUseGoverner()
self.__GovernerThreshhold = Configuration.get().GetGovernerThreshhold()
self.__GovernerBackoffPeriod = Configuration.get(
).GetGovernerBackoffPeriod()
self.__GovernerMaxPacketsBeforeRest = Configuration.get(
).GetGovernerMaxPacketsBeforeRest()
self.__PacketSentSinceRest = 0
self.__BytestSentSinceRest = 0
self.__ShuntedDataByFile = {}
self.__ShuntLock = threading.Lock()
self.__ShuntThreadCreated = False
self.__DownstreamPacketQueue = []
self.__DownstreamPacketLock = threading.Lock()
self.__DownstreamPacketThreads = 0
self.__DownstreamPacketThreadsLock = threading.Lock()
def main():
if not HandleCommandlineArguments():
return
if not Configuration.get().ReadConfigFile():
return
PrintVersion()
downstreamConnInfo = Configuration.get().GetDownstreamConnection()
upstreamConnInfo = Configuration.get().GetUpstreamConnection()
downstreamServer = ServerUDP.ServerUDP(downstreamConnInfo,ConnectionType.DownstreamServer)
upstreamServer = ServerUDP.ServerUDP(upstreamConnInfo,ConnectionType.UpstreamServer)
# if None == Configuration.get().GetAutorunFilename() or 1==1:
if upstreamServer.Start():
if None == downstreamConnInfo:
return
if None == upstreamConnInfo:
return
ThreadManager.GetThreadManager().CreateThread("StartupStuff",StartupWorkerProc,(downstreamServer,upstreamServer))
if not Configuration.get().GetUseGUI():
GuiMgr.Initialize(GuiMgr.UI.NONE,downstreamServer,upstreamServer)
else:
try:
GuiMgr.Initialize(GuiMgr.UI.TKINTR,downstreamServer,upstreamServer)
except Exception as Ex:
print(str(Ex))
GuiMgr.Initialize(GuiMgr.UI.NONE,downstreamServer,upstreamServer)
ThreadManager.GetThreadManager().StartThread("StartupStuff")
GuiMgr.Start()
ThreadManager.GetThreadManager().StopAllThreads()
def __init__(self,Namespace,ID,Value,ElapsedTime,FormatVersion,isLive=True):
from Helpers import Configuration
self.FormatVersion=FormatVersion
self.Value = Value
if True == isLive:
self.ArrivalTime = Time.GetCurrMS() - MarvinData.__FirstTime # just a delta from start is fine, no need for a 10 digit string for time
else:
self.ArrivalTime = ElapsedTime
self.Namespace = Configuration.get().HandleBITWNamespace(Namespace) # if Bump in the Wire, change NS
self.ID = ID
self.Live = isLive
def __CheckForTimeout(self):
from Helpers import Configuration
if not self.m_CanTimeout:
return False
if self.m_hasTimedOut:
return True
toPeriod = Configuration.get().GetTimeoutPeriod()
if toPeriod < Time.GetCurrMS(
) - self.m_lastHeartbeat: # a timeout has ocurred
self.m_hasTimedOut = True
Log.getLogger().info("Target [" + str(self) + "] has timed out.")
self.lastRefreshRequestID = 0
#self.ReArmRefreshRequest(123456789)
return self.m_hasTimedOut
def __init__(self):
self.updateList = []
self.root = Tk()
useTab = True
self._LastDataPointCount = 0
if useTab:
self.TabPane = ttk.Notebook(self.root)
self.tabFrameData = ttk.Frame(self.TabPane)
self.tabFrameStatistics = ttk.Frame(self.TabPane)
self.setupStatisticsTab(self.tabFrameStatistics)
self.TabPane.add(self.tabFrameData, text="Data")
self.TabPane.add(self.tabFrameStatistics, text="Statistics")
self.TabPane.grid(row=0, sticky=(N, S, E, W))
self.TabPane.rowconfigure(
0, weight=1) # Makes it so the data frame will grow
self.TabPane.columnconfigure(
0, weight=1) # Makes it so the data frame will grow
else:
self.tabFrameData = ttk.Frame(self.root)
self.tabFrameData.grid(
row=0, column=0,
sticky=(N, W, E, S)) #Sticky says to grow in those directions,
# self.tabFrame1.rowconfigure(0,weight=1)
# self.tabFrame1.columnconfigure(0,weight=1)
self.tabFrameData.rowconfigure(0, weight=1)
self.tabFrameData.columnconfigure(0, weight=1)
self.setupDataTab(self.tabFrameData)
#Makes tabFrame1 frame grow to size of app
self.root.rowconfigure(0, weight=1)
self.root.columnconfigure(0, weight=1)
self._menu = MenuSystem(self.root)
self.root.config(menu=self._menu.get())
self.updateList.append(self._menu)
if Configuration.get().GetMinimizeGui():
self.root.iconify()
def ReadFromFile(self, filename):
from Helpers import GuiMgr
import os.path
if not os.path.isfile(filename):
Log.getLogger().error("Asked to read from non-existant file: " +
filename)
return False
with open(filename, 'r+b') as fp:
try:
entries = pickle.load(fp)
except Exception as ex:
Log.getLogger().error(filename + ": " + str(ex))
GuiMgr.MessageBox_Error(
"Error Loading File",
"The format of this file requires Python 3.4 or greater.")
return False
if len(entries) < 1:
Log.getLogger().error(
filename + " does not appear to have any data in it. ")
return False
objEntry = entries[0]
if hasattr(objEntry, 'FormatVersion'):
entries = Configuration.get().HandleBITW_FromLoad(entries)
self.SetData(entries)
return True
elif hasattr(objEntry, 'PacketNumber'):
newList = self.ConvertFromAlphaFormat(entries)
else:
newList = self.ConvertFromBetaFormat(entries)
if None != newList:
self.SetData(newList)
return True
return False
def __CollectSingleRange(self,fnKillSignalled,processThreadID):
from Helpers import Configuration
count = 0
currTotal = 0
maxTx = Configuration.GetMaxTransmitBufferBeforeRest()
startTime = Time.GetCurrMS()
collectorList = self.__GetCollectorListForThreadGroup(processThreadID)
for collector in collectorList:
if fnKillSignalled(): # get out of possible long loop if we are to exit
return
if not collector.IsInGroup() and not collector.IsOnDemand():
SizeOfSentData = collector.alternateCollectionProc()
if SizeOfSentData > 0:
self.IncrementSentBytes(SizeOfSentData)
count+=1
currTotal += SizeOfSentData
if currTotal > maxTx: # don't want to overload Oscar
Sleep.SleepMs(50)
currTotal = 0
#timeTaken = Time.GetCurrMS() - startTime
#print(processThreadID +": " + str(timeTaken))
#if Namespace._LogTimePerProcessLoop and timeTaken > 0:
# Log.getLogger().debug("Process Thread: " + collectorList[0].GetProcessThreadID() + " took " + str(timeTaken) + "ms to process one loop")
#if timeTaken > Namespace._LoopTimePeriodWarningThreshold and not Namespace._LogTimePerProcessLoop :
# Log.getLogger().warning("Process Thread: " + collectorList[0].GetProcessThreadID() + " took " + str(timeTaken) + "ms to process one loop - you may want to investigate.")
#if "Default" != processThreadID:
# print(processThreadID + " Collected " + str(count) +"/" + str(len(collectorList)))
return count
def HandleCommandlineArguments():
parser = argparse.ArgumentParser(description='Oscar the wonderful')
parser.add_argument("-i","--input",help='specifies application configuration file file',type=str)
parser.add_argument("-l","--logfile",help='specifies log file name',type=str)
parser.add_argument("-v","--verbose",help="prints debug information",action="store_true")
parser.add_argument("-m","--minimize",help='run with GUI minimized',action="store_true")
exclustionGroup = parser.add_mutually_exclusive_group()
exclustionGroup.add_argument("-p","--playback",help='specifies file to load and playback',type=str)
exclustionGroup.add_argument("-r","--record",help='specifies file to record to, used with --time',type=str)
group_Play = parser.add_argument_group('Playback',"Parameters to be used when you use the --playback option")
group_Play.add_argument("-s","--speed",help='specifies payback speed',type=float)
foo = group_Play.add_mutually_exclusive_group()
foo.add_argument("-ex","--exit",help="exit after playback finished,not valid with repeat or loop",action="store_true")
foo.add_argument("-rp","--repeat",help="repeat the dataset continously",action="store_true")
foo.add_argument("-lp","--loop",help="loop from one datalocation to the next repeatedly (use -begin and -end)",action="store_true")
group_Play.add_argument("-b","--begin",help='start dataset number for mode=loop',default=0,type=int)
group_Play.add_argument("-e","--end",help='end dataset number for mode=loop',type=int)
parser.add_argument("-t","--time",help='specifies time (in minutes) to run before automatically exiting, used with Recording and Playback',type=int)
parser.add_argument("-ng","--nogui",help='run without GUI',action="store_true")
parser.add_argument("-bc","--batchconvert",help="batch convert biff files to csv",type=str)
try:
args = parser.parse_args()
except:
return False
_Verbose = args.verbose
Alias.AliasMgr.AddEnvironmentVariables()
if None != args.logfile:
Configuration.get().SetLogFilename(args.logfile)
if True == _Verbose:
Log.setLevel(logging.DEBUG)
if None != args.batchconvert:
performBatchConvert(args.batchconvert)
return False
conf = Configuration.get()
if True == args.minimize:
conf.SetMinimizeGui(True)
if None != args.input:
if existFile(args.input):
conf.SetConfigFilename(args.input)
else:
return False
if None != args.playback:
if existFile(args.playback):
conf.SetAutorunFilename(args.playback)
conf.SetExitAfterAutoPlay(args.exit)
else:
return False
if None != args.time:
Configuration.get().SetAutorunTime(args.time)
if None == args.playback and None == args.record:
print("time option only valid when doing a playback or a recording")
return false
if None != args.record:
Configuration.get().SetRecordFilename(args.record)
if None != args.speed:
conf.SetPlaybackSpeed(args.speed)
conf.SetAutrunLocations(args.begin,args.end)
if args.repeat:
conf.SetAutoRunMode(RepeatMode.REPEAT)
elif args.loop:
conf.SetAutoRunMode(RepeatMode.LOOP)
elif None != args.playback:
conf.SetAutoRunMode(RepeatMode.NONE)
conf.SetUseGUI(not args.nogui)
return True
def main():
if not HandleCommandlineArguments():
return
if not Configuration.get().ReadConfigFile():
return
PrintVersion()
downstreamConnInfo = Configuration.get().GetDownstreamConnection()
upstreamConnInfo = Configuration.get().GetUpstreamConnection()
downstreamServer = Server.ServerUDP(downstreamConnInfo,
ConnectionType.DownstreamServer)
upstreamServer = Server.ServerUDP(upstreamConnInfo,
ConnectionType.UpstreamServer)
proxyServer = None
proxyClient = None
goodToGo = True
if upstreamServer.Start():
if None == downstreamConnInfo:
return
if None == upstreamConnInfo:
return
ThreadManager.GetThreadManager().CreateThread(
"StartupStuff", StartupWorkerProc,
(downstreamServer, upstreamServer))
if not Configuration.get().GetUseGUI():
GuiMgr.Initialize(GuiMgr.UI.NONE, downstreamServer, upstreamServer)
else:
try:
GuiMgr.Initialize(GuiMgr.UI.TKINTR, downstreamServer,
upstreamServer)
except Exception as Ex:
print(str(Ex))
GuiMgr.Initialize(GuiMgr.UI.NONE, downstreamServer,
upstreamServer)
if None != Configuration.get().GetProxyConnection():
proxyConnInfo_server = Configuration.get().GetProxyConnection()
proxyServer = Server.ServerTCP(
proxyConnInfo_server, ConnectionType.ProxyConnection_Server,
downstreamServer, upstreamServer)
if False == proxyServer.Start():
goodToGo = False
elif None != Configuration.get().GetProxyServerConnection():
proxyConnInfo_Client = Configuration.get(
).GetProxyServerConnection()
proxyClient = Server.ClientTCP(
proxyConnInfo_Client, ConnectionType.ProxyConnection_Client,
downstreamServer, upstreamServer)
if False == proxyClient.Start():
goodToGo = False
if True == goodToGo:
ThreadManager.GetThreadManager().StartThread("StartupStuff")
GuiMgr.Start()
ThreadManager.GetThreadManager().StopAllThreads()
def __workerProc(self, fnKillSignalled, userData):
from Helpers import GuiMgr
self.CurrentIndex = self.startIndex
self.StartTime = None
xmlList = []
while not fnKillSignalled(
): # run until signalled to end - call passed function to check for the signal
if self.Paused or self.Stopped:
Sleep.SleepMs(100)
continue
if None == self.StartTime:
self.StartTime = int(
self.PlaybackData[self.CurrentIndex].ArrivalTime
) - 10 # can't remember why I subract 10ms...
objData = self.PlaybackData[self.CurrentIndex]
sleepVal = (int(objData.ArrivalTime) -
self.StartTime) / self.PlaybackSpeed
Sleep.SleepMs(sleepVal)
try: # when looping, this data will be here after the 1st loop
xmlData = self.PlaybackData[self.CurrentIndex].xmlData
node = self.PlaybackData[self.CurrentIndex].firstNode
except:
xmlData = objData.ToXML(
) # be more efficient if I create a list of already created xmldata
self.PlaybackData[
self.
CurrentIndex].xmlData = xmlData # LOVe how you can just add stuff to an object in Python!
if Configuration.get().GetShunting():
try:
dom = xml.dom.minidom.parseString(rawData)
node = dom._get_firstChild()
except Exception as ex:
Log.getLogger().error(
"Error Something bad in Trying to re-encode saved data"
)
else:
node = None
self.PlaybackData[self.CurrentIndex].firstNode = node
TargetManager.GetTargetManager().BroadcastDownstream(
xmlData, False, node)
GuiMgr.OnDataPacketSentDownstream(objData, "Playback")
try:
self.StartTime = int(
self.PlaybackData[self.CurrentIndex].ArrivalTime)
except Exception:
self.StartTime = None # just in case get an out of bounds error
self.CurrentIndex += 1
if None == self.endIndex:
self.endIndex = len(self.PlaybackData) - 1
if self.CurrentIndex >= self.endIndex:
preProcessingDone = True
if self.LoopMode == RepeatMode.NONE:
GuiMgr.OnStopPlayback()
if Configuration.get().GetExitAfterAutoPlay():
Log.getLogger().info(
"Playback finished, exiting application per arguments"
)
GuiMgr.Quit()
elif self.LoopMode == RepeatMode.REPEAT:
self.CurrentIndex = 0
self.LoopCount += 1
elif self.LoopMode == RepeatMode.LOOP:
self.CurrentIndex = self.startIndex
self.LoopCount += 1
self.StartTime = None
def _ShuntWorker(self, Namespace, ID, Value):
from Helpers import Configuration
config = Configuration.get()
Namespace = config.HandleBITWNamespace(Namespace)
lowerNS = Namespace.lower()
lowerID = ID.lower()
key = lowerNS + lowerID
try:
if key in config.GetNotShuntedMap():
return # already know this one isn't shunted
elif key in config.GetResolvedShuntMap():
self.Shunt(Namespace, ID,
config.GetResolvedShuntMap()[key], Value)
return
elif lowerNS in config.GetShuntMap():
nsMap = config.GetShuntMap()[lowerNS]
if id in nsMap: # matches a shunt, but wasn't in resolved
dataTuple = nsMap[id]
config.GetResolvedShuntMap()[key] = dataTuple
self.Shunt(Namespace, ID, dataTuple, Value)
return
for idKey, dataTuple in nsMap.items():
idPattern = dataTuple[3]
matched = idPattern.match(lowerID)
if None != matched:
#Found a match!
dataTuple = nsMap[idKey]
config.GetResolvedShuntMap()[key] = dataTuple
self.Shunt(Namespace, ID, dataTuple, Value)
return
else:
for nsKey in config.GetShuntMap().keys():
nsPattern = re.compile(nsKey)
matched = nsPattern.match(lowerNS)
if None != matched: # Matched on RegEx Namespace, now check for ID
nsMap = config.GetShuntMap()[nsKey]
if id in nsMap: # ID isn't a RegEx
dataTuple = nsMap[id]
config.GetResolvedShuntMap()[key] = dataTuple
self.Shunt(Namespace, ID, dataTuple, Value)
return
# see if ID is a regEx match
for idKey, dataTuple in nsMap.items():
idPattern = dataTuple[3]
matched = idPattern.match(lowerID)
if None != matched:
#Found a match!
dataTuple = nsMap[idKey]
config.GetResolvedShuntMap()[key] = dataTuple
self.Shunt(Namespace, ID, dataTuple, Value)
return
config.GetNotShuntedMap(
)[key] = key # didn't match any filters, so add to a map so we don't do all the checking
# again
except Exception as Ex:
Log.getLogger().error("Unknown error in _ShuntWorker: " + str(Ex))
def main():
parser = argparse.ArgumentParser(description='Minion Data Collector.')
parser.add_argument("-i",
"--input",
dest='argFilename',
help='specifies input file',
type=extant_file,
metavar="FILE")
parser.add_argument("-v",
"--verbose",
help="prints information, values 0-3",
type=int)
parser.add_argument("-r",
"--runonce",
help="calls all collectors once and exits",
action="store_true")
parser.add_argument(
"-a",
"--aliasfile",
help="specify an external file that has alias defintions",
type=str)
try:
args = parser.parse_args()
if None == args.verbose:
_VerboseLevel = 0
else:
_VerboseLevel = args.verbose
_RunOnce = args.runonce
except:
return
ShowVersion()
if not VersionCheck.CheckVersion():
Log.getLogger().error("Invalid version of Python")
return
if 3 <= _VerboseLevel:
Log.setLevel(logging.DEBUG)
elif 2 == _VerboseLevel:
Log.setLevel(logging.WARNING)
elif 1 == _VerboseLevel:
Log.setLevel(logging.INFO)
else:
Log.setLevel(logging.ERROR)
curr_dir_path = os.path.dirname(os.path.realpath(__file__))
Alias.AliasMgr.AddAlias("WORKING_DIR", curr_dir_path)
Alias.AliasMgr.AddEnvironmentVariables()
if None != args.aliasfile:
if not Alias.AliasMgr.LoadExternalAliasFile(args.aliasfile):
return
signal.signal(
signal.SIGINT, signal.SIG_IGN
) # turn of Ctrl+C signal handler (will get inherted by sub processes
if not os.path.exists(_ConfigFilename):
Log.getLogger().error("Config file [" + _ConfigFilename +
"] not found!")
return
config = Configuration.Configuration(_ConfigFilename, True)
if None == config or not config.IsValid():
pass
else:
print("Starting Collectors...")
totalCollectors = 0
for namespace in config.GetNamespaces():
totalCollectors += namespace.Begin(_RunOnce)
signal.signal(signal.SIGINT,
signal_handler) # make my own Ctrl+C handler now
print(str(totalCollectors) + " Collectors started.")
if False == _RunOnce:
print("Press CTRL+C to Exit")
else:
print("Running Once")
if False == _RunOnce:
while _ThreadActive:
if 0 == _VerboseLevel:
for c in spinning_cursor():
countStr = '[' + str(config.GetCollectorCount()) + '] '
sys.stdout.write(countStr)
sys.stdout.write(c)
Sleep.SleepMs(100)
sys.stdout.flush()
sys.stdout.write('\b')
for c in countStr:
sys.stdout.write('\b')
else:
Sleep.SleepMs(100)
print("Shutting down...")
try:
ThreadManager.GetThreadManager().StopAllThreads()
except:
pass
def HandleOscarTask(self, node, rawData):
#<?xml version="1.0" encoding="utf-8"?>
#<Marvin Type="OscarTask">
# <Version>1.0</Version>
# <OscarID>DemoOscar</OscarID>
# <UniqueID>1233456</UniqueID>
# <Task>taskid</Task>
# <Param>p1</Param>
#</Marvin>
Statistics.GetStatistics().OnOscarTaskReceived()
try:
UniqueID = node.getElementsByTagName(
'UniqueID')[0].firstChild.nodeValue
OscarID = node.getElementsByTagName(
'OscarID')[0].firstChild.nodeValue
task = node.getElementsByTagName('Task')[0].firstChild.nodeValue
tTask = task.lower()
Params = []
for param in node.getElementsByTagName(
'Param'): # Make an array of the params for the script
strParam = param.firstChild.nodeValue
Params.append(strParam)
except Exception as ex:
Statistics.GetStatistics().OnMalformedPacketReceived(
"Received bad Oscar Task: " + rawData)
if OscarID.lower() != Configuration.get().GetID().lower(
): # Oscar packet, but probably not destined for this Oscar
if OscarID.lower() == "Broadcast".lower():
Log.getLogger().info("Received Broadcast Oscar task: " + task)
else:
Log.getLogger().info("Received Oscar task [" + task +
"] for another Oscar with ID: " +
OscarID) # not for this guy
#increment dropped stats?
return
if UniqueID == self.LastUniqueID:
#Log.getLogger().info("Dropping duplicate Oscar Task")
return
self.LastUniqueID = UniqueID
if tTask == "StartLive".lower():
self.PerformGoLiveTask(Params)
elif tTask == "StopLive".lower():
self.PerformStopLiveTask(Params)
elif tTask == "LoadFile".lower():
self.PerformLoadFileTask(Params)
elif tTask == "Playback".lower():
self.PerformPlayFileTask(Params)
elif tTask == "StopPlayback".lower():
self.PerformStopPlaybackTask(Params)
elif tTask == "PausePlayback".lower():
self.PerformPausePlaybackTask(Params)
elif tTask == "StartRecording".lower():
self.PerformStartRecordingTask(Params)
elif tTask == "StopRecording".lower():
self.PerformStopRecordingTask(Params)
else:
Log.getLogger().warn("Unknown Oscar Task: " + task)
def GetConfiguration():
if None == Configuration._ConfigurationInstance:
Configuration._ConfigurationInstance = Configuration()
return Configuration._ConfigurationInstance
def GetNamespace(strNamespaceID):
from Helpers import Configuration
return Configuration.GetNamespace(strNamespaceID)