def doMessageAction(obj, msg, messaging=None):
"""
The function takes a message, and demuxxes it. Based on the content of the message it
may take a number of actions. That number is currently one: invoke dispatchCall
which calls a function on "this" object whatever it is.
"""
log.debug("In doMessageAction %s %s", str(obj), str(msg))
log.debug("Content type: %d", msg.contenttype)
#First deserialize the message and then switch on the action.
if msg.contenttype == MAGIMessage.PICKLE:
log.debug("Content type: Pickle")
data = cPickle.loads(msg.data)
else:
# Default data type is YAML
data = yaml.load(msg.data)
if 'method' in data:
try:
retVal = dispatchCall(obj, msg, data)
except Exception, e:
log.error("Agent %s threw an exception %s during main loop",
str(obj),
e,
exc_info=1)
log.error(
"Sending back a RunTimeException event. This may cause the receiver to exit."
)
exc_type, exc_value, exc_tb = sys.exc_info()
filename, line_num, func_name, text = traceback.extract_tb(
exc_tb)[-1]
filename = basename(filename)
messaging.trigger(event='RuntimeException',
func_name=func_name,
agent=str(obj),
nodes=[config.getNodeName()],
filename=filename,
line_num=line_num,
error=str(e))
return
if 'trigger' in data:
if not isinstance(retVal, dict):
if retVal is None:
retVal = True
retVal = {'retVal': retVal}
args = retVal
args['nodes'] = config.getNodeName()
agentName = getattr(obj, 'name')
if agentName:
args['agent'] = agentName
messaging.trigger(event=data['trigger'], **args)
def initClient(self, msg):
log.info("Initializing DMM client...")
self.collection = database.getCollection(self.name)
self.collection.remove()
nodeAssignment = None
with open(os.path.join(self.configPath, "nodeAssignment.json"), 'r') as assignmentFile:
nodeAssignment = json.load(assignmentFile)
# nodeName: "clietnode-3" --> nodeIndex: 3
self.nodeIndex = int(getNodeName().split("-")[1])
# clientId ==> maps int nodeIndex to string like "DC-8"
splitstr = self.clientID.split('-')
self.clientType = splitstr[0]
self.clientIdx = int(splitstr[1])
jsonVals = json.load(
os.path.join(self.configPath, "fullcase.json")
)
clientIdxStr = str(self.clientIdx)
self.PMax = jsonVals[self.clientType][clientIdxStr]["PMax"]
self.PMin = jsonVals[self.clientType][clientIdxStr]["PMin"]
self.b = jsonVals[self.clientType][clientIdxStr]["b"]
self.c = jsonVals[self.clientType][clientIdxStr]["c"]
self.M = jsonVals[self.clientType][clientIdxStr]["M"]
if self.clientType=='G':
self.delta=jsonVals[stype][clientIdxStr]["delta"]
self.gamma=jsonVals[stype][clientIdxStr]["gamma"]
return True
def initClient(self, msg):
log.info("Initializing DMM client...")
self.collection = database.getCollection(self.name)
self.collection.remove()
nodeAssignment = None
with open(os.path.join(self.configPath, "nodeAssignment.json"),
'r') as assignmentFile:
nodeAssignment = json.load(assignmentFile)
# nodeName: "clietnode-3" --> nodeIndex: 3
self.nodeIndex = int(getNodeName().split("-")[1])
# clientId ==> maps int nodeIndex to string like "DC-8"
splitstr = self.clientID.split('-')
self.clientType = splitstr[0]
self.clientIdx = int(splitstr[1])
jsonVals = json.load(os.path.join(self.configPath, "fullcase.json"))
clientIdxStr = str(self.clientIdx)
self.PMax = jsonVals[self.clientType][clientIdxStr]["PMax"]
self.PMin = jsonVals[self.clientType][clientIdxStr]["PMin"]
self.b = jsonVals[self.clientType][clientIdxStr]["b"]
self.c = jsonVals[self.clientType][clientIdxStr]["c"]
self.M = jsonVals[self.clientType][clientIdxStr]["M"]
if self.clientType == 'G':
self.delta = jsonVals[stype][clientIdxStr]["delta"]
self.gamma = jsonVals[stype][clientIdxStr]["gamma"]
return True
def isDBRunning(host='localhost', port=DATABASE_SERVER_PORT):
"""
Check if a database server is running on a given host and port
"""
#In case of a single node experiment /etc/hosts does not get populated
if host == config.getNodeName():
host = 'localhost'
return Server.isDBRunning(host=helpers.toControlPlaneNodeName(host),
port=port)
def doMessageAction(obj, msg, messaging=None):
"""
The function takes a message, and demuxxes it. Based on the content of the message it
may take a number of actions. That number is currently one: invoke dispatchCall
which calls a function on "this" object whatever it is.
"""
log.debug("In doMessageAction %s %s", str(obj), str(msg))
log.debug("Content type: %d", msg.contenttype)
#First deserialize the message and then switch on the action.
if msg.contenttype == MAGIMessage.PICKLE:
log.debug("Content type: Pickle")
data = cPickle.loads(msg.data)
else:
# Default data type is YAML
data = yaml.load(msg.data)
if 'method' in data:
try:
retVal = dispatchCall(obj, msg, data)
except Exception, e:
log.error("Agent %s threw an exception %s during main loop", str(obj), e, exc_info=1)
log.error("Sending back a RunTimeException event. This may cause the receiver to exit.")
exc_type, exc_value, exc_tb = sys.exc_info()
filename, line_num, func_name, text = traceback.extract_tb(exc_tb)[-1]
filename = basename(filename)
messaging.trigger(event='RuntimeException', func_name=func_name, agent=str(obj),
nodes=[config.getNodeName()], filename=filename, line_num=line_num, error=str(e))
return
if 'trigger' in data:
if not isinstance(retVal, dict):
if retVal is None:
retVal = True
retVal = {'retVal' : retVal}
args = retVal
args['nodes'] = config.getNodeName()
agentName = getattr(obj, 'name')
if agentName:
args['agent'] = agentName
messaging.trigger(event=data['trigger'], **args)
def ping(self, msg):
"""
Alive like method call that will send a pong back to the caller
"""
args = {
"server": config.getNodeName(),
"result": "success"
}
call = {'version': 1.0, 'method': 'pong', 'args': args}
msg = MAGIMessage(nodes=msg.src, docks='dataman', contenttype=MAGIMessage.YAML, data=yaml.dump(call))
self.messenger.send(msg)
def ping(self, msg):
"""
Alive like method call that will send a pong back to the caller
"""
args = {"server": config.getNodeName(), "result": "success"}
call = {'version': 1.0, 'method': 'pong', 'args': args}
msg = MAGIMessage(nodes=msg.src,
docks='dataman',
contenttype=MAGIMessage.YAML,
data=yaml.dump(call))
self.messenger.send(msg)
def registerShard(
mongod=config.getNodeName(), mongos=getConfigHost(), timeout=TIMEOUT):
"""
Function to register a database server as a shard
in the database cluster
"""
functionName = registerShard.__name__
helpers.entrylog(log, functionName, locals())
mongod = helpers.toControlPlaneNodeName(mongod)
mongos = helpers.toControlPlaneNodeName(mongos)
Server.registerShard(dbHost=mongod, configHost=mongos, timeout=timeout)
helpers.exitlog(log, functionName)
def initClient(self, msg):
log.info("Initializing client...")
self.collection = database.getCollection(self.name)
self.collection.remove()
# nodeName: "clientnode-3" --> nodeIndex: 3
self.nodeIndex = int(getNodeName().split("-")[1])
with open(self.configFileName, 'r') as configFile:
globalConfig = json.load(configFile)
unitConfig = globalConfig["units"][self.nodeIndex-1]
self.CID = unitConfig["CID"]
self.unit = BBB_ISO.dictToUnit(unitConfig)
self.unit.tS = globalConfig["timeStep"]
self.t = 0
def format(self, record):
"""Format exception object as a string"""
data = record.__dict__.copy()
if record.args:
record.msg = record.msg % record.args
data.update(
host=config.getNodeName(),
message=record.msg,
username=getpass.getuser(),
time=datetime.now(),
args=tuple(unicode(arg) for arg in record.args)
)
if 'exc_info' in data and data['exc_info']:
data['exc_info'] = self.formatException(data['exc_info'])
return data
def getCollection(agentName, dbHost=None, dbPort=DATABASE_SERVER_PORT):
"""
Function to get a pointer to a given agent data collection
"""
functionName = getCollection.__name__
helpers.entrylog(log, functionName, locals())
if dbHost == None:
dbHost = getCollector()
global collectionHosts
collectionHosts[agentName].add(dbHost)
connection = getConnection(host=dbHost, port=dbPort, block=False)
helpers.exitlog(log, functionName)
return Collection.getCollection(agentName=agentName,
hostName=config.getNodeName(),
connection=connection)
def getConnection(host=None,
port=DATABASE_SERVER_PORT,
block=True,
timeout=TIMEOUT):
"""
Function to get connection to a database server
"""
functionName = getConnection.__name__
helpers.entrylog(log, functionName, locals())
if host == None:
host = getCollector()
#In case of a single node experiment /etc/hosts does not get populated
if host == config.getNodeName():
host = 'localhost'
helpers.exitlog(log, functionName)
return Connection.getConnection(host=helpers.toControlPlaneNodeName(host),
port=port,
block=block,
timeout=timeout)
def isCollector():
sensorToCollectorMap = getSensorToCollectorMap()
return (config.getNodeName() in sensorToCollectorMap.values())
def initializeProcessAgent(agent, argv):
'''argv is assumed to have the following format. (This is usually set by the
Magi daemon):
agent_name agent_dock execute=[pipe|socket] (logfile=path)
Where agent_name and agent_dock are strings and the key in the key=value
pairs is literally the key given. The value may be restricted.
'''
if len(argv) < 3:
log.critical('command line must start with name and dock')
sys.exit(2)
agent.name, dock, nodeConfigFile, experimentConfigFile = argv[1:5]
args = argv_to_dict(argv[5:])
config.loadNodeConfig(nodeConfigFile, experimentConfigFile)
setAttributes(
agent, {
'hostname':
config.getNodeName(),
'execute':
'socket',
'logfile':
os.path.join(config.getLogDir(), agent.name + '.log'),
'loglevel':
'DEBUG',
'commHost':
'localhost',
'commPort':
config.getConfig()['localInfo'].get('processAgentsCommPort',
18809),
'commGroup':
None
}, args)
agent.docklist.add(dock)
helpers.makeDir(os.path.dirname(agent.logfile))
handler = logging.FileHandler(agent.logfile, 'w')
handler.setFormatter(
logging.Formatter(helpers.LOG_FORMAT_MSECS, helpers.LOG_DATEFMT))
root = logging.getLogger()
root.setLevel(helpers.logLevels.get(agent.loglevel.lower(), logging.INFO))
root.handlers = []
root.addHandler(handler)
log.info('argv: %s', argv)
log.info('agent attributes: %s', agent.__dict__)
log.info("Setting up agent messaging interface")
inTransport, outTransport = _getIOHandles(agent)
agent.messenger = AgentMessenger(inTransport, outTransport, agent)
agent.messenger.start()
log.info("Agent messaging interface initialized and running")
# Tell the daemon we want to listen on the dock.
# GTL - why doesn't the Daemon just associate the dock
# with this process?
agent.messenger.listenDock(dock)
if agent.commGroup:
agent.messenger.joinGroup(agent.commGroup)
#TODO: In ideal condition wait from the node to join group before proceeding further
# now that we're connected, send an AgentLoaded message.
agent.messenger.trigger(event='AgentLoadDone',
agent=agent.name,
nodes=[agent.hostname])
return args
def isConfigHost():
configHost = getConfigHost()
return (config.getNodeName() == configHost or
helpers.toControlPlaneNodeName(config.getNodeName()) == configHost)
def isCollector(self, node, agentName):
"""
Internal function to check if the local node is the collector for a given node and agent
"""
return (self.getCollector(node, agentName) == config.getNodeName())
helpers.makeDir(os.path.dirname(config.getNodeConfFile()))
helpers.writeYaml(nodeConfig, config.getNodeConfFile())
log.info("Created a node configuration file at %s",
config.getNodeConfFile())
helpers.makeDir(os.path.dirname(config.getExperimentConfFile()))
helpers.writeYaml(experimentConfig, config.getExperimentConfFile())
log.info("Created a experiment configuration file at %s",
config.getExperimentConfFile())
except:
log.exception(
"MAGI configuration failed, things probably aren't going to run"
)
if (config.getNodeName() == config.getServer()):
import shutil
log.info(
"Copying experiment.conf to testbed experiment directory %s" %
(config.getExperimentDir()))
shutil.copy(config.getExperimentConfFile(),
config.getExperimentDir())
# Now that the system is configured, import database library
from magi.util import database
if database.isDBEnabled:
if (database.isCollector or database.isConfigHost):
if not options.noinstall:
#installPackage('mongodb', 'mongodb') #Package installer starts mongodb with default configuration
#Copying prebuilt binaries for mongodb
log.info("Created a node configuration file at %s",
config.getNodeConfFile())
helpers.makeDir(os.path.dirname(
config.getExperimentConfFile()))
helpers.writeYaml(experimentConfig,
config.getExperimentConfFile())
log.info("Created a experiment configuration file at %s",
config.getExperimentConfFile())
except:
log.exception(
"MAGI configuration failed, things probably aren't going to run"
)
if (config.getNodeName() == helpers.getServer(
config.getMagiNodes())):
import shutil
log.info(
"Copying experiment.conf to testbed experiment directory %s"
% (config.getExperimentDir()))
shutil.copy(config.getExperimentConfFile(),
config.getExperimentDir())
# Now that the system is configured, import database library
from magi.util import database
if database.isDBEnabled():
if (database.isCollector() or database.isConfigHost()):
if not options.noinstall:
#installPackage('mongodb', 'mongodb') #Package installer starts mongodb with default configuration
else:
log.info("Using node configuration file at %s", options.nodeconf)
nodeConfig = config.loadNodeConfig(nodeConfig=options.nodeconf, experimentConfig=experimentConfig)
helpers.makeDir(os.path.dirname(config.getNodeConfFile()))
helpers.writeYaml(nodeConfig, config.getNodeConfFile())
log.info("Created a node configuration file at %s", config.getNodeConfFile())
helpers.makeDir(os.path.dirname(config.getExperimentConfFile()))
helpers.writeYaml(experimentConfig, config.getExperimentConfFile())
log.info("Created a experiment configuration file at %s", config.getExperimentConfFile())
except:
log.exception("MAGI configuration failed, things probably aren't going to run")
if (config.getNodeName() == config.getServer()):
import shutil
log.info("Copying experiment.conf to testbed experiment directory %s" %(config.getExperimentDir()))
shutil.copy(config.getExperimentConfFile(), config.getExperimentDir())
# Now that the system is configured, import database library
from magi.util import database
if database.isDBEnabled:
if (database.isCollector or database.isConfigHost):
if not options.noinstall:
#installPackage('mongodb', 'mongodb') #Package installer starts mongodb with default configuration
#Copying prebuilt binaries for mongodb
if is_os_64bit():
installPreBuilt('mongodb-linux-x86_64')
else:
else:
log.info("Using node configuration file at %s", options.nodeconf)
nodeConfig = config.loadNodeConfig(nodeConfig=options.nodeconf, experimentConfig=experimentConfig)
helpers.makeDir(os.path.dirname(config.getNodeConfFile()))
helpers.writeYaml(nodeConfig, config.getNodeConfFile())
log.info("Created a node configuration file at %s", config.getNodeConfFile())
helpers.makeDir(os.path.dirname(config.getExperimentConfFile()))
helpers.writeYaml(experimentConfig, config.getExperimentConfFile())
log.info("Created a experiment configuration file at %s", config.getExperimentConfFile())
except:
log.exception("MAGI configuration failed, things probably aren't going to run")
if (config.getNodeName() == helpers.getServer(config.getMagiNodes())):
import shutil
log.info("Copying experiment.conf to testbed experiment directory %s" %(config.getExperimentDir()))
shutil.copy(config.getExperimentConfFile(), config.getExperimentDir())
# Now that the system is configured, import database library
from magi.util import database
if database.isDBEnabled():
if (database.isCollector() or database.isConfigHost()):
if not options.noinstall:
#installPackage('mongodb', 'mongodb') #Package installer starts mongodb with default configuration
#Copying prebuilt binaries for mongodb
if is_os_64bit():
installPreBuilt('mongodb-linux-x86_64', '/usr/local/bin/mongo', rpath)
else:
def isSensor():
sensorToCollectorMap = getSensorToCollectorMap()
return (config.getNodeName() in sensorToCollectorMap.keys()
or helpers.DEFAULT in sensorToCollectorMap.keys())
def getCollector():
sensorToCollectorMap = getSensorToCollectorMap()
return sensorToCollectorMap.get(config.getNodeName(),
sensorToCollectorMap.get(helpers.DEFAULT))
def isCollector(self, node, agentName):
"""
Internal function to check if the local node is the collector for a given node and agent
"""
return (self.getCollector(node, agentName) == config.getNodeName())