def doMessageAction(obj, msg, messaging=None): """ The function takes a message, and demuxxes it. Based on the content of the message it may take a number of actions. That number is currently one: invoke dispatchCall which calls a function on "this" object whatever it is. """ log.debug("In doMessageAction %s %s", str(obj), str(msg)) log.debug("Content type: %d", msg.contenttype) #First deserialize the message and then switch on the action. if msg.contenttype == MAGIMessage.PICKLE: log.debug("Content type: Pickle") data = cPickle.loads(msg.data) else: # Default data type is YAML data = yaml.load(msg.data) if 'method' in data: try: retVal = dispatchCall(obj, msg, data) except Exception, e: log.error("Agent %s threw an exception %s during main loop", str(obj), e, exc_info=1) log.error( "Sending back a RunTimeException event. This may cause the receiver to exit." ) exc_type, exc_value, exc_tb = sys.exc_info() filename, line_num, func_name, text = traceback.extract_tb( exc_tb)[-1] filename = basename(filename) messaging.trigger(event='RuntimeException', func_name=func_name, agent=str(obj), nodes=[config.getNodeName()], filename=filename, line_num=line_num, error=str(e)) return if 'trigger' in data: if not isinstance(retVal, dict): if retVal is None: retVal = True retVal = {'retVal': retVal} args = retVal args['nodes'] = config.getNodeName() agentName = getattr(obj, 'name') if agentName: args['agent'] = agentName messaging.trigger(event=data['trigger'], **args)
def initClient(self, msg): log.info("Initializing DMM client...") self.collection = database.getCollection(self.name) self.collection.remove() nodeAssignment = None with open(os.path.join(self.configPath, "nodeAssignment.json"), 'r') as assignmentFile: nodeAssignment = json.load(assignmentFile) # nodeName: "clietnode-3" --> nodeIndex: 3 self.nodeIndex = int(getNodeName().split("-")[1]) # clientId ==> maps int nodeIndex to string like "DC-8" splitstr = self.clientID.split('-') self.clientType = splitstr[0] self.clientIdx = int(splitstr[1]) jsonVals = json.load( os.path.join(self.configPath, "fullcase.json") ) clientIdxStr = str(self.clientIdx) self.PMax = jsonVals[self.clientType][clientIdxStr]["PMax"] self.PMin = jsonVals[self.clientType][clientIdxStr]["PMin"] self.b = jsonVals[self.clientType][clientIdxStr]["b"] self.c = jsonVals[self.clientType][clientIdxStr]["c"] self.M = jsonVals[self.clientType][clientIdxStr]["M"] if self.clientType=='G': self.delta=jsonVals[stype][clientIdxStr]["delta"] self.gamma=jsonVals[stype][clientIdxStr]["gamma"] return True
def initClient(self, msg): log.info("Initializing DMM client...") self.collection = database.getCollection(self.name) self.collection.remove() nodeAssignment = None with open(os.path.join(self.configPath, "nodeAssignment.json"), 'r') as assignmentFile: nodeAssignment = json.load(assignmentFile) # nodeName: "clietnode-3" --> nodeIndex: 3 self.nodeIndex = int(getNodeName().split("-")[1]) # clientId ==> maps int nodeIndex to string like "DC-8" splitstr = self.clientID.split('-') self.clientType = splitstr[0] self.clientIdx = int(splitstr[1]) jsonVals = json.load(os.path.join(self.configPath, "fullcase.json")) clientIdxStr = str(self.clientIdx) self.PMax = jsonVals[self.clientType][clientIdxStr]["PMax"] self.PMin = jsonVals[self.clientType][clientIdxStr]["PMin"] self.b = jsonVals[self.clientType][clientIdxStr]["b"] self.c = jsonVals[self.clientType][clientIdxStr]["c"] self.M = jsonVals[self.clientType][clientIdxStr]["M"] if self.clientType == 'G': self.delta = jsonVals[stype][clientIdxStr]["delta"] self.gamma = jsonVals[stype][clientIdxStr]["gamma"] return True
def isDBRunning(host='localhost', port=DATABASE_SERVER_PORT): """ Check if a database server is running on a given host and port """ #In case of a single node experiment /etc/hosts does not get populated if host == config.getNodeName(): host = 'localhost' return Server.isDBRunning(host=helpers.toControlPlaneNodeName(host), port=port)
def doMessageAction(obj, msg, messaging=None): """ The function takes a message, and demuxxes it. Based on the content of the message it may take a number of actions. That number is currently one: invoke dispatchCall which calls a function on "this" object whatever it is. """ log.debug("In doMessageAction %s %s", str(obj), str(msg)) log.debug("Content type: %d", msg.contenttype) #First deserialize the message and then switch on the action. if msg.contenttype == MAGIMessage.PICKLE: log.debug("Content type: Pickle") data = cPickle.loads(msg.data) else: # Default data type is YAML data = yaml.load(msg.data) if 'method' in data: try: retVal = dispatchCall(obj, msg, data) except Exception, e: log.error("Agent %s threw an exception %s during main loop", str(obj), e, exc_info=1) log.error("Sending back a RunTimeException event. This may cause the receiver to exit.") exc_type, exc_value, exc_tb = sys.exc_info() filename, line_num, func_name, text = traceback.extract_tb(exc_tb)[-1] filename = basename(filename) messaging.trigger(event='RuntimeException', func_name=func_name, agent=str(obj), nodes=[config.getNodeName()], filename=filename, line_num=line_num, error=str(e)) return if 'trigger' in data: if not isinstance(retVal, dict): if retVal is None: retVal = True retVal = {'retVal' : retVal} args = retVal args['nodes'] = config.getNodeName() agentName = getattr(obj, 'name') if agentName: args['agent'] = agentName messaging.trigger(event=data['trigger'], **args)
def ping(self, msg): """ Alive like method call that will send a pong back to the caller """ args = { "server": config.getNodeName(), "result": "success" } call = {'version': 1.0, 'method': 'pong', 'args': args} msg = MAGIMessage(nodes=msg.src, docks='dataman', contenttype=MAGIMessage.YAML, data=yaml.dump(call)) self.messenger.send(msg)
def ping(self, msg): """ Alive like method call that will send a pong back to the caller """ args = {"server": config.getNodeName(), "result": "success"} call = {'version': 1.0, 'method': 'pong', 'args': args} msg = MAGIMessage(nodes=msg.src, docks='dataman', contenttype=MAGIMessage.YAML, data=yaml.dump(call)) self.messenger.send(msg)
def registerShard( mongod=config.getNodeName(), mongos=getConfigHost(), timeout=TIMEOUT): """ Function to register a database server as a shard in the database cluster """ functionName = registerShard.__name__ helpers.entrylog(log, functionName, locals()) mongod = helpers.toControlPlaneNodeName(mongod) mongos = helpers.toControlPlaneNodeName(mongos) Server.registerShard(dbHost=mongod, configHost=mongos, timeout=timeout) helpers.exitlog(log, functionName)
def initClient(self, msg): log.info("Initializing client...") self.collection = database.getCollection(self.name) self.collection.remove() # nodeName: "clientnode-3" --> nodeIndex: 3 self.nodeIndex = int(getNodeName().split("-")[1]) with open(self.configFileName, 'r') as configFile: globalConfig = json.load(configFile) unitConfig = globalConfig["units"][self.nodeIndex-1] self.CID = unitConfig["CID"] self.unit = BBB_ISO.dictToUnit(unitConfig) self.unit.tS = globalConfig["timeStep"] self.t = 0
def format(self, record): """Format exception object as a string""" data = record.__dict__.copy() if record.args: record.msg = record.msg % record.args data.update( host=config.getNodeName(), message=record.msg, username=getpass.getuser(), time=datetime.now(), args=tuple(unicode(arg) for arg in record.args) ) if 'exc_info' in data and data['exc_info']: data['exc_info'] = self.formatException(data['exc_info']) return data
def getCollection(agentName, dbHost=None, dbPort=DATABASE_SERVER_PORT): """ Function to get a pointer to a given agent data collection """ functionName = getCollection.__name__ helpers.entrylog(log, functionName, locals()) if dbHost == None: dbHost = getCollector() global collectionHosts collectionHosts[agentName].add(dbHost) connection = getConnection(host=dbHost, port=dbPort, block=False) helpers.exitlog(log, functionName) return Collection.getCollection(agentName=agentName, hostName=config.getNodeName(), connection=connection)
def getConnection(host=None, port=DATABASE_SERVER_PORT, block=True, timeout=TIMEOUT): """ Function to get connection to a database server """ functionName = getConnection.__name__ helpers.entrylog(log, functionName, locals()) if host == None: host = getCollector() #In case of a single node experiment /etc/hosts does not get populated if host == config.getNodeName(): host = 'localhost' helpers.exitlog(log, functionName) return Connection.getConnection(host=helpers.toControlPlaneNodeName(host), port=port, block=block, timeout=timeout)
def isCollector(): sensorToCollectorMap = getSensorToCollectorMap() return (config.getNodeName() in sensorToCollectorMap.values())
def initializeProcessAgent(agent, argv): '''argv is assumed to have the following format. (This is usually set by the Magi daemon): agent_name agent_dock execute=[pipe|socket] (logfile=path) Where agent_name and agent_dock are strings and the key in the key=value pairs is literally the key given. The value may be restricted. ''' if len(argv) < 3: log.critical('command line must start with name and dock') sys.exit(2) agent.name, dock, nodeConfigFile, experimentConfigFile = argv[1:5] args = argv_to_dict(argv[5:]) config.loadNodeConfig(nodeConfigFile, experimentConfigFile) setAttributes( agent, { 'hostname': config.getNodeName(), 'execute': 'socket', 'logfile': os.path.join(config.getLogDir(), agent.name + '.log'), 'loglevel': 'DEBUG', 'commHost': 'localhost', 'commPort': config.getConfig()['localInfo'].get('processAgentsCommPort', 18809), 'commGroup': None }, args) agent.docklist.add(dock) helpers.makeDir(os.path.dirname(agent.logfile)) handler = logging.FileHandler(agent.logfile, 'w') handler.setFormatter( logging.Formatter(helpers.LOG_FORMAT_MSECS, helpers.LOG_DATEFMT)) root = logging.getLogger() root.setLevel(helpers.logLevels.get(agent.loglevel.lower(), logging.INFO)) root.handlers = [] root.addHandler(handler) log.info('argv: %s', argv) log.info('agent attributes: %s', agent.__dict__) log.info("Setting up agent messaging interface") inTransport, outTransport = _getIOHandles(agent) agent.messenger = AgentMessenger(inTransport, outTransport, agent) agent.messenger.start() log.info("Agent messaging interface initialized and running") # Tell the daemon we want to listen on the dock. # GTL - why doesn't the Daemon just associate the dock # with this process? agent.messenger.listenDock(dock) if agent.commGroup: agent.messenger.joinGroup(agent.commGroup) #TODO: In ideal condition wait from the node to join group before proceeding further # now that we're connected, send an AgentLoaded message. agent.messenger.trigger(event='AgentLoadDone', agent=agent.name, nodes=[agent.hostname]) return args
def isConfigHost(): configHost = getConfigHost() return (config.getNodeName() == configHost or helpers.toControlPlaneNodeName(config.getNodeName()) == configHost)
def isCollector(self, node, agentName): """ Internal function to check if the local node is the collector for a given node and agent """ return (self.getCollector(node, agentName) == config.getNodeName())
helpers.makeDir(os.path.dirname(config.getNodeConfFile())) helpers.writeYaml(nodeConfig, config.getNodeConfFile()) log.info("Created a node configuration file at %s", config.getNodeConfFile()) helpers.makeDir(os.path.dirname(config.getExperimentConfFile())) helpers.writeYaml(experimentConfig, config.getExperimentConfFile()) log.info("Created a experiment configuration file at %s", config.getExperimentConfFile()) except: log.exception( "MAGI configuration failed, things probably aren't going to run" ) if (config.getNodeName() == config.getServer()): import shutil log.info( "Copying experiment.conf to testbed experiment directory %s" % (config.getExperimentDir())) shutil.copy(config.getExperimentConfFile(), config.getExperimentDir()) # Now that the system is configured, import database library from magi.util import database if database.isDBEnabled: if (database.isCollector or database.isConfigHost): if not options.noinstall: #installPackage('mongodb', 'mongodb') #Package installer starts mongodb with default configuration #Copying prebuilt binaries for mongodb
log.info("Created a node configuration file at %s", config.getNodeConfFile()) helpers.makeDir(os.path.dirname( config.getExperimentConfFile())) helpers.writeYaml(experimentConfig, config.getExperimentConfFile()) log.info("Created a experiment configuration file at %s", config.getExperimentConfFile()) except: log.exception( "MAGI configuration failed, things probably aren't going to run" ) if (config.getNodeName() == helpers.getServer( config.getMagiNodes())): import shutil log.info( "Copying experiment.conf to testbed experiment directory %s" % (config.getExperimentDir())) shutil.copy(config.getExperimentConfFile(), config.getExperimentDir()) # Now that the system is configured, import database library from magi.util import database if database.isDBEnabled(): if (database.isCollector() or database.isConfigHost()): if not options.noinstall: #installPackage('mongodb', 'mongodb') #Package installer starts mongodb with default configuration
else: log.info("Using node configuration file at %s", options.nodeconf) nodeConfig = config.loadNodeConfig(nodeConfig=options.nodeconf, experimentConfig=experimentConfig) helpers.makeDir(os.path.dirname(config.getNodeConfFile())) helpers.writeYaml(nodeConfig, config.getNodeConfFile()) log.info("Created a node configuration file at %s", config.getNodeConfFile()) helpers.makeDir(os.path.dirname(config.getExperimentConfFile())) helpers.writeYaml(experimentConfig, config.getExperimentConfFile()) log.info("Created a experiment configuration file at %s", config.getExperimentConfFile()) except: log.exception("MAGI configuration failed, things probably aren't going to run") if (config.getNodeName() == config.getServer()): import shutil log.info("Copying experiment.conf to testbed experiment directory %s" %(config.getExperimentDir())) shutil.copy(config.getExperimentConfFile(), config.getExperimentDir()) # Now that the system is configured, import database library from magi.util import database if database.isDBEnabled: if (database.isCollector or database.isConfigHost): if not options.noinstall: #installPackage('mongodb', 'mongodb') #Package installer starts mongodb with default configuration #Copying prebuilt binaries for mongodb if is_os_64bit(): installPreBuilt('mongodb-linux-x86_64') else:
else: log.info("Using node configuration file at %s", options.nodeconf) nodeConfig = config.loadNodeConfig(nodeConfig=options.nodeconf, experimentConfig=experimentConfig) helpers.makeDir(os.path.dirname(config.getNodeConfFile())) helpers.writeYaml(nodeConfig, config.getNodeConfFile()) log.info("Created a node configuration file at %s", config.getNodeConfFile()) helpers.makeDir(os.path.dirname(config.getExperimentConfFile())) helpers.writeYaml(experimentConfig, config.getExperimentConfFile()) log.info("Created a experiment configuration file at %s", config.getExperimentConfFile()) except: log.exception("MAGI configuration failed, things probably aren't going to run") if (config.getNodeName() == helpers.getServer(config.getMagiNodes())): import shutil log.info("Copying experiment.conf to testbed experiment directory %s" %(config.getExperimentDir())) shutil.copy(config.getExperimentConfFile(), config.getExperimentDir()) # Now that the system is configured, import database library from magi.util import database if database.isDBEnabled(): if (database.isCollector() or database.isConfigHost()): if not options.noinstall: #installPackage('mongodb', 'mongodb') #Package installer starts mongodb with default configuration #Copying prebuilt binaries for mongodb if is_os_64bit(): installPreBuilt('mongodb-linux-x86_64', '/usr/local/bin/mongo', rpath) else:
def isSensor(): sensorToCollectorMap = getSensorToCollectorMap() return (config.getNodeName() in sensorToCollectorMap.keys() or helpers.DEFAULT in sensorToCollectorMap.keys())
def getCollector(): sensorToCollectorMap = getSensorToCollectorMap() return sensorToCollectorMap.get(config.getNodeName(), sensorToCollectorMap.get(helpers.DEFAULT))
def isCollector(self, node, agentName): """ Internal function to check if the local node is the collector for a given node and agent """ return (self.getCollector(node, agentName) == config.getNodeName())