def Start(self):
if None != self.m_Name:
return
self.m_Name = str(self)
self.m_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
try:
self.m_socket.bind((self.getIP(), self.getPort()))
self.m_socket.setblocking(True)
self.m_socket.settimeout(0.01)
except Exception:
Log.getLogger().error("Invalid Socket IP or Port " + str(self))
return False
if 0 == self.getPort(): # let the OS choose the port
self.__Port = self.m_socket.getsockname()[
1] #can use this to pass to Marvin
self.__Port = self.__Port # ungly kludge
self.__IP = self.__IP # ungly kludge
self._objNamespace.__ListenPort = self.__Port
Log.getLogger().debug("Namespace[" + str(self._objNamespace) +
"] listening on -->" + str(self))
ThreadManager.GetThreadManager().CreateThread(self.m_Name,
self.workerProc)
ThreadManager.GetThreadManager().StartThread(self.m_Name)
def updateGui(self):
targets = TargetManager.GetTargetManager().GetDownstreamTargets()
# was a channge in number (maybe a dynamci marvin went away) so just clear the tree and re-poplualte
if len(targets) != self.PreviousTargetCount:
self.PreviousTargetCount = len(targets)
self.tree.delete(*self.tree.get_children())
for key in targets:
target = TargetManager.GetTargetManager().GetDownstreamTarget(key)
strPackets = str(target.m_PacketsSent)
strBytes = str(target.m_BytestSent)
strType = target.getTypeStr()
try:
self.tree.set(key, 'Packets', strPackets)
self.tree.set(key, 'Bytes', strBytes)
self.tree.set(key, 'Type', strType)
if True == target.m_hasTimedOut:
self.tree.set(key, 'IP', "*" + target.getIP())
else:
self.tree.set(key, 'IP', target.getIP())
except Exception as Ex:
try:
self.tree.insert('',
'end',
key,
values=(target.getIP(),
str(target.getPort()), strType,
strPackets, strBytes))
except Exception as Ex:
Log.getLogger().error(str(Ex))
def workerProc(self, fnKillSignalled, userData):
dataHandler = DataHandler.GetDataHandler()
from Helpers import Configuration
buffSize = Configuration.get().GetRecvBufferSize()
try:
while not fnKillSignalled(
): # run until signalled to end - call passed function to check for the signal
try:
data, fromAddress = self.m_socket.recvfrom(buffSize)
data = data.strip().decode("utf-8")
self.m_rxPackets += 1
self.m_rxBytes += len(data)
except: # socket.error:
#time.sleep(.01) #no data, so sleep for 10ms
continue
if not fnKillSignalled():
if False == self.__DropPackets():
dataHandler.HandleLiveData(data, fromAddress)
elif "<Marvin Type=\"Bullhorn\">" in data:
dataHandler.HandleLiveData(
data,
fromAddress) # drop all but the Marvin Bullhorn
except Exception as ex:
Log.getLogger().debug("Thread Error: " + str(ex) + " --> " +
traceback.format_exc())
def Alias(input):
orig = input
index = input.find("$(") #Alias is surrounded by -->$( ) <--
stopIndex = input[index:].find(")") + index
if stopIndex < index:
otherIndex = input[index:].find(")")
stopIndex = index + otherIndex
while index != -1 and stopIndex != -1: # do it in a loop in case there are more than one alias
strAlias = input[index + 2:stopIndex]
if AliasMgr.IsAliased(strAlias):
newStr = ""
newStr = newStr + input[0:index] + AliasMgr.GetAlias(
strAlias) + input[stopIndex + 1:len(input)]
input = newStr
index = input.find("$(")
stopIndex = input[index:].find(")") + index
else:
Log.getLogger().warn(
"Something looks like an Alias, but there is no alias registered for it --> "
+ input)
return orig
return input
def LatestLogEntry():
global MyLogger
if None == MyLogger:
MyLogger = MyHandler()
Log.getLogger().addHandler(MyLogger)
return MyLogger.getLatest()
def __BoundData(self,sendValue):
# self._Bound_Max=None
# self._Bound_Min=None
# self._Bound_Action=BoundAction.Invalid
if None == self._Bound_Max and None == self._Bound_Min:
return sendValue
try:
value = float(sendValue)
except Exception:
Log.getLogger().warning("Collector [" + self.GetID() + "] tried to perform data bounding, but data is not numeric.")
return sendValue
if None != self._Bound_Min and value < self._Bound_Min:
if self._Bound_Action == BoundAction.Set:
returnVal = self._Bound_Min
elif self._Bound_Action == BoundAction.RepeatLast:
returnVal = self._LastSentValue
else:
returnVal = None
elif None != self._Bound_Max and value > self._Bound_Max:
if self._Bound_Action == BoundAction.Set:
returnVal = self._Bound_Max
elif self._Bound_Action == BoundAction.RepeatLast:
returnVal = self._LastSentValue
else:
returnVal = None
else:
returnVal = sendValue
returnVal = self.__AssignPrecisionAndScale(returnVal,True)
return returnVal
def __CreateSendBuffer(self, value, elapsedtime, normalized):
if None == value: #whoa, this should not happen
Log.getLogger().error("Asked to send a non existant value. ID=" +
self.GetID())
return None
if len(str(value)) == 0:
Log.getLogger().warn("Collector [" + self.GetID() +
"] returned empty string. Dropping.")
return None
if None == self._NamespaceOverride:
namespaceStr = str(self._NamespaceObject)
else:
namespaceStr = self._NamespaceOverride
buffer = ""
buffer = buffer + "<Minion Type=\"Data\">"
buffer = buffer + "<Version>1</Version>"
buffer = buffer + "<PacketNumber>" + str(
self._NamespaceObject.getNextPacketNumber()) + "</PacketNumber>"
buffer = buffer + "<Namespace>" + namespaceStr + "</Namespace>"
buffer = buffer + "<ID>" + self.GetTransmitID() + "</ID>"
buffer = buffer + "<Value>" + value + "</Value>"
buffer = buffer + "<Normalized>" + str(normalized) + "</Normalized>"
buffer = buffer + "<ElapsedTime>" + str(elapsedtime) + "</ElapsedTime>"
buffer = buffer + "</Minion>"
return buffer
def HandleMinionTask(self,node,rawData,objNamespace):
#<?xml version="1.0" encoding="utf-8"?>
#<Marvin Type="MinionTask">
# <Version>1.0</Version>
# <UniqueID>1233456</UniqueID>
# <Task Namespace="fuBar" ID="RunMyTask"/>
# <Param>p1</Param>
#</Marvin>
from Helpers import Namespace
objTempActor = Actor.Actor()
try:
objTempActor.LastUniqueID = node.getElementsByTagName('UniqueID')[0].firstChild.nodeValue
taskNode = node.getElementsByTagName('Task')[0]
attributes = taskNode.attributes
objTempActor.Namespace = Alias.Alias(attributes["Namespace"].nodeValue)
objTempActor.ID = Alias.Alias(attributes["ID"].nodeValue)
for param in node.getElementsByTagName("Param"): # Make an array of the params for the script
strParam = Alias.Alias(param.firstChild.nodeValue)
objTempActor.Parameters.append(strParam)
except Exception as ex:
Log.getLogger().error("Error Parsing Minion Task: " + rawData + " : " + str(ex))
return
objNamespace.Enact(objTempActor)
def _VerifyInput(self,CompareType):
if self._Verified:
return True
collectors = self.GetCollectors()
if len(collectors) == 0:
return "Operator -Input- still pending"
if len(collectors) < 3:
if not self._WarningSent:
Log.getLogger().warn(CompareType + " Operator does not have enough input.")
return 'Invalid " + CompareType + " Configuration'
if len(collectors) > 4:
Log.getLogger().warn(CompareType + " Operator has too many inputs")
return 'Invalid " + CompareType + " Configuration'
self._Value1 = collectors[0]
self._Value2 = collectors[1]
self._If = collectors[2]
if len(collectors) == 4:
self._Else = collectors[3]
self._ReallyDoNotSend = self._DoNotSend
if None != self._Collectors:
self._Verified = True
return True
def renameId(args):
inpFiles = glob.glob(g_args.input)
totalRenamedPoints = 0
totalIdsChanged = 0
fCount = 0
for inpName in inpFiles:
fHandler = Actions.FileHandler(inpName)
pointInFileChanged = 0
idsInFileChanged = 0
for namespace in args.namespace:
pointCount, idCount = fHandler.Rename_Id(namespace, args.id,
args.new)
pointInFileChanged += pointCount
idsInFileChanged += idCount
targetFn = GetTargetFileName(inpName, g_args.output)
fHandler.writeFile(targetFn, g_args.overwrite)
Log.getLogger().info("{} IDs, {} dataponts renamed in {}".format(
idsInFileChanged, pointInFileChanged, inpName))
totalRenamedPoints += pointInFileChanged
totalIdsChanged += idsInFileChanged
if idsInFileChanged > 0:
fCount += 1
Log.getLogger().info("Renamed {} ids in {} files, {} datapoints".format(
totalIdsChanged, fCount, totalRenamedPoints))
def parseAndRunAction(actionName, actionList, argList, usageList=""):
if isinstance(actionList, ArgObject): # is an argparse object
argObj = actionList
parser = argObj.getParser()
args = argObj.getParser().parse_args(argList)
try:
argObj.getWorkerFn()(args)
except Exception as Ex:
Log.getLogger().error(str(Ex))
return
for currAction in actionList:
if currAction.getAction() == actionName:
if None != currAction.getParser():
args = currAction.getParser().parse_args(argList)
break
else: #is a list of args
validOptions = currAction.getActionStrings()
requestedAction = [argList[0]]
otherArgs = argList[1:]
parser = argparse.ArgumentParser()
parser.add_argument(
actionName,
help=actionName + ' to perform',
nargs="?",
choices=validOptions,
)
parser.parse_args(requestedAction)
for actionItem in currAction.getActionList():
if actionItem.getAction() == requestedAction[
0]: # requestedAction is in array for argPArse
return parseAndRunAction(requestedAction, actionItem,
otherArgs)
def StopThread(self,strName):
objThread = self.__GetThread(strName)
if None != objThread:
objThread.Stop()
else:
Log.getLogger().error("StartThread called with unknown thread name: " + strName)
def AddCollector(self, objCollector, beforeID=None):
if objCollector.GetID().lower() in self._CollectorMap:
Log.getLogger().error("Duplicate Collector found: " +
objCollector.GetID())
return False
#Collectors are in a MAP for fast retrieval
self._CollectorMap[objCollector.GetID().lower()] = objCollector
# Dynamic Collectors should be inserted right AFTER the
# DynamicCollector collector, otherwise if appended to the end,
# operators that use data from
# a dynamic collector will be run using stale data
if None == beforeID:
self._Collectors.append(objCollector)
else:
InsertAfterInList(self._Collectors, beforeID, objCollector)
if objCollector.GetProcessThreadID(
) in self.__ProcessThreadGroupings.keys():
InsertAfterInList(
self.__ProcessThreadGroupings[
objCollector.GetProcessThreadID()], beforeID,
objCollector)
else:
Log.getLogger().error("Not supposed to end up here!")
return True
def ApplyDelta_Id(self, namespaceName, ids, deltaVal):
namespaces = self.getMatchingNamespacesNameList(namespaceName)
totalModifiedCount = 0
if not isinstance(ids, list):
ids = [ids]
if len(namespaces) > 0:
for namespace in namespaces:
for searchId in ids:
for entry in self._namespaceMap[namespace]:
if isinstance(entry, MarvinGroupData.MarvinDataGroup):
for subEntry in entry._DataList:
if Matches(subEntry.ID, searchId):
if DeltaValue(subEntry, deltaVal):
totalModifiedCount += 1
elif Matches(entry.ID.lower(), searchId):
if DeltaValue(entry, deltaVal):
totalModifiedCount += 1
else:
Log.getLogger().error(
"Namespace: {} does not exist".format(namespaceName))
return totalModifiedCount
def _BroadcastDownstream(self,
sendBuffer,
ignoreTimeout,
domNode,
isGroup=False):
from Helpers import Configuration
sentCount = 0
for targetKey in self._DownstreamTargets.keys():
if True == self.SendToDownstreamTarget(sendBuffer, targetKey,
ignoreTimeout):
sentCount += 1
if sentCount > 0:
Statistics.GetStatistics().OnPacketBroadcastDownstream()
if Configuration.get().GetShunting() and (None != domNode
or True == isGroup):
if None != domNode and not isGroup:
self.HandleShuntingData(domNode)
else: # is a group and need to go through each item individually
if None == domNode:
try:
dom = xml.dom.minidom.parseString(sendBuffer)
domNode = dom._get_firstChild()
except Exception as ex:
Log.getLogger().error(
"Error Something bad in Trying to read xml in BroadcastDownstream"
)
if None != domNode:
for dataNode in domNode.getElementsByTagName('Oscar'):
self.HandleShuntingData(dataNode)
return sentCount
def Collect(self):
list = self.GetCollectors()
if len(list) < 1:
return None
try:
for objCollector in list:
if not hasattr(objCollector, "MaxCollectedValue"):
objCollector.MaxCollectedValue = objCollector.GetLastValue()
if not Utility.IsNumeric(list[0].MaxCollectedValue):
if not self._InvalidInpWarningSent:
Log.getLogger().warn("An Input to Operator MaxValue is non numeric.--> " + objCollector.MaxCollectedValue)
self._InvalidInpWarningSent = True
return "HelenKeller"
max = float(list[0].MaxCollectedValue)
for objCollector in list:
if not Utility.IsNumeric(objCollector.MaxCollectedValue):
return None
val = float(objCollector.MaxCollectedValue)
if not objCollector.ReadyForConsumption() and not objCollector.IsDefaultValue():
return None # hasn't yet collecte
if val > max:
max = val
except Exception as Ex:
if not self._InvalidInpWarningSent:
Log.getLogger().warn("An Input to Operator MaxValue is non numeric.--> " + objCollector.MaxCollectedValue)
self._InvalidInpWarningSent = True
max = ""
return str(max)
def Shunt(self, namespace, ID, dataTuple, Value):
Statistics.GetStatistics().OnPacketShunted()
newTuple = (namespace, ID, dataTuple[4], Value)
shuntFile = dataTuple[2]
self.__ShuntLock.acquire()
try:
if not shuntFile in self.__ShuntedDataByFile:
self.__ShuntedDataByFile[shuntFile] = []
self.__ShuntedDataByFile[shuntFile].append(newTuple)
except Exception as Ex:
Log.getLogger().info("Unknown in Shunt function: " + str(Ex))
finally:
self.__ShuntLock.release()
if not self.__ShuntThreadCreated:
self.__ShuntThreadCreated = True
threadName = "ShuntProc:" + str(self)
ThreadManager.GetThreadManager().CreateThread(
threadName, self.ShuntWorkerProc)
ThreadManager.GetThreadManager().StartThread(threadName)
def SetCurrentNumber(self,newNumber):
if Utility.IsNumeric(newNumber) and newNumber> -1 and newNumber < self.GetDataCount():
self.CurrentIndex = newNumber
self.IndexExternallySet = True
else:
Log.getLogger().warn("Tried to set playback index to invalid value: " + str(newNumber))
def __Scale(self, value):
try:
float(value)
try:
return self.__AssignPrecisionAndScale(value)
except Exception as Ex:
Log.getLogger().warning("Collector: " + self.GetID() +
" tried to scale : " + value)
return value
except:
pass
#is an array
newValue = []
try:
valList = value.split(",") #split at commas
except:
return value # wast not a float, nor an array, so could be an operator for a dynamic collector or dynamic collector with no data yet
for item in valList:
try:
newValue.append(self.__AssignPrecisionAndScale(item))
except:
#Log.getLogger().warning("tried to scale : " + value)
return value
return ",".join(newValue)
def Start(self):
if len(self.PlaybackData) == 0:
Log.getLogger().error("Tried to play empty playback data.")
return
if None == self.endIndex or self.endIndex > len(self.PlaybackData) - 1:
self.endIndex = len(self.PlaybackData) - 1
if True == self.Stopped:
if self.startIndex > self.endIndex:
Log.getLogger().error(
"Start Index > End Index for Playback, ignoring")
self.startIndex = 0
self.endIndex = len(self.PlaybackData) - 1
return
if False == self.IndexExternallySet:
self.CurrentIndex = self.startIndex
self.StartTime = None #let worker calculate
self.IndexExternallySet = False
self.Stopped = False
elif True == self.Paused:
self.Paused = False
def alternateCollectionProc(self):
if self.IsOnDemand():
Log.getLogger().error(
"On Demand Collector called with alternateCollectionProc")
if self.NeedsCollecting():
startCollectionTime = Time.GetCurrMS()
buffer = self.PerformCollection()
TimeToCollect = Time.GetCurrMS() - startCollectionTime
#print(TimeToCollect)
if None != buffer:
buffer = "<?xml version=\"1.0\" encoding=\"utf-8\"?>" + buffer
if not self._NamespaceObject.SendPacket(buffer):
return 0
#self._NamespaceObject.CheckMTU(len(buffer),self._MinionID)
if TimeToCollect > self._PollingInterval: # do a sanity check to see if collection time is longer than collector frequency
Log.getLogger().warning(
"Collector: " + self.GetID() +
" took longer to perform the actual collection than the specified frequency for it ("
+ str(self._PollingInterval) +
". It is suggested you change something to fix this.")
return len(buffer)
return 0
def SetData(self, dataList):
if False == self.Stopped:
Log.getLogger().error(
"Setting Playback data, but playback has not been stopped")
return
self.PlaybackData = dataList
self.LoopMode = RepeatMode.NONE
self.LoopCount = 1
self.StartTime = None
self.CurrentIndex = 0
self.startIndex = 0
self.endIndex = len(dataList)
NamespaceMap = {}
self.NamespaceCount = 0
self.ID_Count = 0
# go get some stats on the loaded data
for entry in dataList:
if not entry.Namespace in NamespaceMap:
NamespaceMap[entry.Namespace] = {}
self.NamespaceCount += 1
if hasattr(entry, '_DataList'):
for subEntry in entry._DataList:
if not subEntry.ID in NamespaceMap[entry.Namespace]:
NamespaceMap[entry.Namespace][
subEntry.ID] = subEntry.ID
self.ID_Count += 1
elif not entry.ID in NamespaceMap[entry.Namespace]:
NamespaceMap[entry.Namespace][entry.ID] = entry.ID
self.ID_Count += 1
def getNamespacesFromBuffer(self, buffer):
from Helpers import Configuration
config = Configuration.get()
start = '<Namespace>'
end = '</Namespace>'
reg = "(?<=%s).*?(?=%s)" % (start, end)
try:
result = re.search(reg, buffer)
r = re.compile(reg)
NamespaceList = r.findall(buffer)
if None == NamespaceList:
Log.getLogger().error(
"Error Something bad in trying to find namespaces HandleBITW "
+ buffer)
return None
list = []
map = config.GetBITW_NamespaceMap()
checkedMap = {}
for namespace in NamespaceList:
if not namespace in checkedMap:
checkedMap[
namespace] = namespace # could be a group, and only need to check once, but could be case differences, so don't use upper
strUp = namespace.upper()
if strUp in map:
list.append((namespace, map[strUp]))
except Exception as ex:
print(str(ex))
return list
def PerformStopRecordingTask(self, Params):
#<?xml version="1.0" encoding="utf-8"?>
#<Marvin Type="OscarTask">
# <Version>1.0</Version>
# <OscarID>DemoOscar</OscarID>
# <Task>StopRecording</Task>
# <Param>File=SaveFile.glk</Param>
#</Marvin>
fileName = None
param = ""
if len(Params) > 0:
param = Params[0]
parts = param.split("=")
if len(parts) == 2:
if parts[0].lower() == 'file':
fileName = Alias.Alias(parts[1])
if None == fileName:
Log.getLogger().error(
"Received invalid Stop Recording task. No save file: " +
str(param))
return
GuiMgr.OnStopRecording()
GuiMgr.WriteToFile(fileName)
def updateGui(self):
try:
dlist = GuiMgr.get().GetDatalist()
for key in dlist.keys():
try:
objData = dlist[key][0]
strFrom = dlist[key][1]
self.dataViewTree.set(key, 'Value', objData.Value)
self.dataViewTree.set(key, 'Source', strFrom)
except Exception as Ex:
try:
index = self.__findIndex(key)
self.dataViewTree.insert('',
index,
key,
values=(objData.Namespace,
objData.ID,
str(objData.Value),
strFrom))
except Exception as Ex:
Log.getLogger().error(str(Ex))
except Exception as Ex: # Likely had the list updated while I was iterating (didn't make this thread safe), just ignore and wait for next loop
pass
def __NormalizeArray(self, newValue, timeDelta):
try:
splitData = newValue.split(",")
except Exception:
Log.getLogger().error(
"Tried to normalize an array that wasn't an array: " +
newValue)
return newValue
oldData = self._LastValue.split(",")
index = 0
retString = None
if len(splitData) == 0: #wasn't comma separated data
return newValue
for dataPoint in splitData:
try:
valueDelta = float(dataPoint) - float(oldData[index])
except Exception:
return newValue #tried to normalize non-int data, so just return what we got and forget it
dataRatePerSec = float(
float(valueDelta) / float(float(timeDelta) / 1000.0))
dataRateWithNormFactor = dataRatePerSec * float(
self._NormalizeValue)
index += 1
if None == retString:
retString = str(dataRateWithNormFactor)
else:
retString += "," + str(dataRateWithNormFactor)
return retString #should be a comma separated normalized data
def AddAlias(Alias,Value):
strAlias = Alias.upper()
if strAlias in AliasMgr.__AliasList[0]:
Log.getLogger().warning("Attempt to create duplicate alias key: " + strAlias)
return
AliasMgr.__AliasList[0][strAlias] = Value # only add to the top list
def __AlternateCollectionMethodMultiThread(self,fnKillSignalled,startIndex):
processedWithoutRestdummymakelooklikeother = 0
ThreadCount = -1
ProcessThreadCount = self.__CreateInitialCollectorThreadGroupings()
AddActiveProcessingThreads(ProcessThreadCount)
firstGroupID = None
firstGroupCollectors = []
collectorCount = len(self._Collectors)
if collectorCount < 1:
Log.getLogger().error("No Collectors to process")
return
for processThreadID,collectorList in self.__ProcessThreadGroupings.items():
if None == firstGroupID:
firstGroupID = processThreadID
firstGroupCollectors = collectorList
else:
ID = str(self) + processThreadID
ThreadManager.GetThreadManager().CreateThread(ID,self.__SlicedThreadProc,processThreadID) # create a worker thread and pass it a list of collectors to update
ThreadManager.GetThreadManager().StartThread(ID)
ThreadCount += 1
while not fnKillSignalled(): # now go process the 1st group in this thread
processed = self.__CollectSingleRange(fnKillSignalled,firstGroupID)
if processed == 0:
Sleep.SleepMs(Namespace.SleepIntervalIfNoDataCollected)
if collectorCount != len(self._Collectors): # dynamic collectos must have added some
pass
def Start(self):
if None != self.m_Name:
return
self.m_Name = "ServerUDP:" + str(self)
self.m_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
try:
self.m_socket.bind((self.getIP(), self.getPort()))
self.m_socket.setblocking(True)
self.m_socket.settimeout(0.001)
except Exception as ex:
Log.getLogger().error("Invalid Socket IP or Port " + str(self) +
" - " + str(ex))
return False
if 0 == self.getPort(): # let the OS choose the port
self.Port = self.m_socket.getsockname()[
1] #can use this to pass to Marvin
self.m_ConnPoint.Port = self.Port # ungly kludge
self.m_ConnPoint.IP = self.IP # ungly kludge
Log.getLogger().info(self.getTypeStr() + " listening on -->" +
str(self))
ThreadManager.GetThreadManager().CreateThread(self.m_Name,
self.workerProc)
ThreadManager.GetThreadManager().StartThread(self.m_Name)
return True
def Collect(self):
list = self.GetCollectors()
if len(list) < 1:
return None
try:
for objCollector in list:
if not hasattr(objCollector, "MinCollectedValue"):
objCollector.MinCollectedValue = objCollector._LastValue
min = float(list[0].MinCollectedValue)
for objCollector in list:
val = float(objCollector.MinCollectedValue)
if not objCollector.ReadyForConsumption():
return "" # hasn't yet collected
if val < min:
min = val
except Exception as Ex:
Log.getLogger().warn(
"An Input to Operator MinValue is non numeric.")
min = ""
return str(min)
