def run(self):
"""
Called by daemon in the agent's thread to perform the thread main
"""
self.logfile = os.path.join(config.getLogDir(), '%s_%s.log' % (self.name, time.strftime("%Y-%m-%d_%H:%M:%S")))
if database.isDBEnabled():
log.info("Database enabled. Creating collection instance.")
self.collection = database.getCollection(self.name)
while not self.done:
try:
msg = self.messenger.next(True, max(self.nextrun - time.time(), 0))
if isinstance(msg, MAGIMessage):
doMessageAction(self, msg, self.messenger)
except Queue.Empty:
pass
if not self.running or time.time() < self.nextrun:
continue # message received, not yet time to launch, reloop
self.nextrun = time.time() + eval(self.interval)
try:
# TODO: Check memory here to avoid overload?
self.oneClient()
except Exception:
log.error("error in client process", exc_info=1)
def run(self):
"""
Called by daemon in the agent's thread to perform the thread main
"""
self.logfile = os.path.join(
config.getLogDir(),
'%s_%s.log' % (self.name, time.strftime("%Y-%m-%d_%H:%M:%S")))
if database.isDBEnabled():
log.info("Database enabled. Creating collection instance.")
self.collection = database.getCollection(self.name)
while not self.done:
try:
msg = self.messenger.next(True,
max(self.nextrun - time.time(), 0))
if isinstance(msg, MAGIMessage):
doMessageAction(self, msg, self.messenger)
except Queue.Empty:
pass
if not self.running or time.time() < self.nextrun:
continue # message received, not yet time to launch, reloop
self.nextrun = time.time() + eval(self.interval)
try:
# TODO: Check memory here to avoid overload?
self.oneClient()
except Exception:
log.error("error in client process", exc_info=1)
def startShardServer(configHost=getConfigHost(), timeout=TIMEOUT):
"""
Function to start a database config server on the node
"""
return Server.startShardServer(
logPath=os.path.join(config.getLogDir(), "mongos.log"),
configHost=helpers.toControlPlaneNodeName(configHost),
timeout=timeout)
def startConfigServer(timeout=TIMEOUT):
"""
Function to start a database config server on the node
"""
return Server.startConfigServer(
dbPath=os.path.join(config.getDbDir(), "configdb"),
logPath=os.path.join(config.getLogDir(), "mongoc.log"),
timeout=timeout)
def doAndLogCommand(self, command, name):
'''Run the given command and dump stdout and stderr to the log directory.'''
# TODO: replace hardcoded file name with magi.util.config.MAGILOG
filename = os.path.join(config.getLogDir(), '%s-%s.log' % (self.TYPE, name))
try:
with open(filename, 'a') as fd:
return run(command, stdout=fd, stderr=subprocess.STDOUT, shell=True) == 0
except Exception, e:
self.log.warning('%s failed %s: %s. Check %s for details.', self.TYPE, name, e, filename)
return False
def startDBServer(configfile=None, timeout=TIMEOUT):
"""
Function to start a database server on the node
"""
helpers.makeDir(config.getDbDir())
return Server.startDBServer(configfile=configfile,
configDir=config.getConfigDir(),
dbPath=os.path.join(config.getDbDir(),
"mongodb"),
logPath=os.path.join(config.getLogDir(),
"mongodb.log"),
timeout=timeout)
def archive(self, msg, destinationDir=config.getTempDir()):
"""
Tars the log directory"
"""
functionName = self.archive.__name__
helpers.entrylog(log, functionName, locals())
logDir = config.getLogDir()
logTar = tarfile.open(name=os.path.join(destinationDir,
"logs_%s.tar.gz"%(datetime.datetime.now()
.strftime("%Y%m%d_%H%M%S"))),
mode='w:gz')
logTar.add(logDir, arcname=os.path.basename(logDir))
logTar.close()
helpers.exitlog(log, functionName)
def getLogsArchive(self, msg):
"""
Tars the log directory and sends it to the requester as a message"
"""
functionName = self.archive.__name__
helpers.entrylog(log, functionName, locals())
logDir = config.getLogDir()
store = cStringIO.StringIO()
logTar = tarfile.open(fileobj=store, mode='w:gz')
logTar.add(logDir, arcname=os.path.basename(logDir))
logTar.close()
result = base64.encodestring(store.getvalue())
self.messaging.send(MAGIMessage(nodes=msg.src, docks=msg.srcdock,
contenttype=MAGIMessage.YAML,
data=yaml.safe_dump(result)))
helpers.exitlog(log, functionName)
def startAgent(self, code=None, name=None, dock=None, execargs=None, idl=None, static=False):
"""
Internal function to invoke an agent
"""
# Now find the interface definition and load it
try:
log.debug('startAgent code: %s, idl: %s' % (code, idl))
dirname = code
if idl:
idlFile = dirname+'/%s.idl' % idl
else:
idlFile = glob.glob(dirname+'/*.idl')[0]
except IndexError:
log.debug("No valid interface file in %s", dirname)
raise OSError(errno.ENOENT, "No valid interface file found in %s" % dirname)
log.debug('reading interface file %s...' % idlFile)
fp = open(idlFile)
interface = fp.read()
fp.close()
interface = yaml.load(interface)
# If there are software dependencies, load them before loading the agent
if 'software' in interface:
for package in interface['software']:
log.info('Loading required package %s for agent %s.', package, name)
requireSoftware(package)
compileCmd = interface.get('compileCmd')
if compileCmd:
log.info("Running specified compilation command: '%s' under directory '%s'" %(compileCmd, dirname))
p = Popen(compileCmd.split(), cwd=dirname, stdout=PIPE, stderr=PIPE)
if p.wait():
raise OSError("Exception while running the specified compilation command. %s", p.communicate())
# Based on the interface execution method, execute the agent
execstyle = interface['execute']
mainfile = os.path.join(dirname, interface['mainfile'])
log.info('Running agent from file %s' % mainfile)
# GTL TODO: handle exceptions from threaded agents by removing
# the agents and freeing up the dock(s) for the agent.
try:
if execstyle == 'thread':
# A agent should know the hostname and its own name
from magi.daemon.threadInterface import ThreadedAgent
agent = ThreadedAgent(self.hostname, name, mainfile, dock, execargs, self.messaging)
agent.start()
log.info("Started threaded agent %s", agent)
if static:
self.staticAgents.append(agent)
else:
self.threadAgents.append(agent)
#2/13/14 Moved it to threadInterface
#self.messaging.trigger(event='AgentLoadDone', agent=name, nodes=[self.hostname])
else:
# Process agent, use the file as written to disk
if (not execargs) or (type(execargs) != dict):
execargs = dict()
# Process agent need to know hostname
execargs['hostname'] = self.hostname
# I apologize for this abuse
args = ['%s=%s' % (str(k), yaml.dump(v)) for k,v in execargs.iteritems()]
os.chmod(mainfile, 00777)
stderrname = os.path.join(config.getLogDir(), name + '.stderr')
stderr = open(stderrname, 'w') # GTL should this be closed? If so, when?
log.info("Starting %s, stderr sent to %s", name, stderrname)
if execstyle == 'pipe':
args.append('execute=pipe')
cmdList = [mainfile, name, dock, config.getNodeConfFile(), config.getExperimentConfFile()] + args
log.info('running: %s', ' '.join(cmdList))
agent = Popen(cmdList, close_fds=True, stdin=PIPE, stdout=PIPE, stderr=stderr)
self.extAgentsThread.fromNetwork.put(PipeTuple([dock], InputPipe(fileobj=agent.stdout), OutputPipe(fileobj=agent.stdin)))
elif execstyle == 'socket':
args.append('execute=socket')
cmdList = [mainfile, name, dock, config.getNodeConfFile(), config.getExperimentConfFile()] + args
log.info('running: %s', ' '.join(cmdList))
agent = Popen(cmdList, close_fds=True, stderr=stderr)
else:
log.critical("unknown launch style '%s'", interface['execute'])
return False
self.pAgentPids[name] = agent.pid
#Process agent once up, sends the AgentLoadDone message itself
#self.messaging.trigger(event='AgentLoadDone', agent=name, nodes=[self.hostname] )
except Exception, e:
log.error("Agent %s on %s threw an exception %s during agent load.", name, self.hostname, e, exc_info=1)
log.error("Sending back a RunTimeException event. This may cause the receiver to exit.")
exc_type, exc_value, exc_tb = sys.exc_info()
filename, line_num, func_name, text = traceback.extract_tb(exc_tb)[-1]
filename = basename(filename)
self.messaging.trigger(event='RuntimeException', type=exc_type.__name__, error=str(e), nodes=[self.hostname],
agent=self.name, func_name=func_name, filename=filename, line_num=line_num)
return False
def initializeProcessAgent(agent, argv):
'''argv is assumed to have the following format. (This is usually set by the
Magi daemon):
agent_name agent_dock execute=[pipe|socket] (logfile=path)
Where agent_name and agent_dock are strings and the key in the key=value
pairs is literally the key given. The value may be restricted.
'''
if len(argv) < 3:
log.critical('command line must start with name and dock')
sys.exit(2)
agent.name, dock, nodeConfigFile, experimentConfigFile = argv[1:5]
args = argv_to_dict(argv[5:])
config.loadNodeConfig(nodeConfigFile, experimentConfigFile)
setAttributes(
agent, {
'hostname':
config.getNodeName(),
'execute':
'socket',
'logfile':
os.path.join(config.getLogDir(), agent.name + '.log'),
'loglevel':
'DEBUG',
'commHost':
'localhost',
'commPort':
config.getConfig()['localInfo'].get('processAgentsCommPort',
18809),
'commGroup':
None
}, args)
agent.docklist.add(dock)
helpers.makeDir(os.path.dirname(agent.logfile))
handler = logging.FileHandler(agent.logfile, 'w')
handler.setFormatter(
logging.Formatter(helpers.LOG_FORMAT_MSECS, helpers.LOG_DATEFMT))
root = logging.getLogger()
root.setLevel(helpers.logLevels.get(agent.loglevel.lower(), logging.INFO))
root.handlers = []
root.addHandler(handler)
log.info('argv: %s', argv)
log.info('agent attributes: %s', agent.__dict__)
log.info("Setting up agent messaging interface")
inTransport, outTransport = _getIOHandles(agent)
agent.messenger = AgentMessenger(inTransport, outTransport, agent)
agent.messenger.start()
log.info("Agent messaging interface initialized and running")
# Tell the daemon we want to listen on the dock.
# GTL - why doesn't the Daemon just associate the dock
# with this process?
agent.messenger.listenDock(dock)
if agent.commGroup:
agent.messenger.joinGroup(agent.commGroup)
#TODO: In ideal condition wait from the node to join group before proceeding further
# now that we're connected, send an AgentLoaded message.
agent.messenger.trigger(event='AgentLoadDone',
agent=agent.name,
nodes=[agent.hostname])
return args
help="Specify location of the experiment " +
"configuration file, ex: -c experiment.conf ")
optparser.add_option("-c", "--nodeconf", dest="nodeconf",
help="Specify location of the node " +
"configuration file, ex: -c localnode.conf ")
(options, args) = optparser.parse_args()
expConfig = config.loadExperimentConfig(options.expconf)
nodeConfig = config.loadNodeConfig(options.nodeconf)
#import once the system is cofigured
from magi.daemon.daemon import Daemon
if not options.logfile:
options.logfile = os.path.join(config.getLogDir(), "daemon.log")
helpers.makeDir(os.path.dirname(options.logfile))
helpers.makeDir(config.getTempDir())
# Roll over the old log and create a new one
# Note here that we will have at most 5 logs
# Need to check existence of file before creating the handler instance
# This is because handler creation creates the file if not existent
if os.path.isfile(options.logfile):
needroll = True
else:
needroll = False
handler = logging.handlers.RotatingFileHandler(options.logfile, backupCount=5)
if needroll:
handler.doRollover()
optparser.add_option("-c",
"--nodeconf",
dest="nodeconf",
help="Specify location of the node " +
"configuration file, ex: -c localnode.conf ")
(options, args) = optparser.parse_args()
expConfig = config.loadExperimentConfig(options.expconf)
nodeConfig = config.loadNodeConfig(options.nodeconf)
#import once the system is cofigured
from magi.daemon.daemon import Daemon
if not options.logfile:
options.logfile = os.path.join(config.getLogDir(), "daemon.log")
helpers.makeDir(os.path.dirname(options.logfile))
helpers.makeDir(config.getTempDir())
# Roll over the old log and create a new one
# Note here that we will have at most 5 logs
# Need to check existence of file before creating the handler instance
# This is because handler creation creates the file if not existent
if os.path.isfile(options.logfile):
needroll = True
else:
needroll = False
handler = logging.handlers.RotatingFileHandler(options.logfile,
backupCount=5)
if needroll: