def startRun(self, runSet, runNum, runOptions, logDir=None):
if logDir is None:
logDir = self.__defaultLogDir
try:
openCount = self.__countFileDescriptors()
except:
self.__log.error("Cannot count open files: %s" % exc_string())
openCount = 0
runSet.startRun(runNum,
self.getClusterConfig().configName(),
runOptions,
get_version_info(SVN_ID),
self.__spadeDir,
copyDir=self.__copyDir,
logDir=logDir,
quiet=self.__quiet)
if self.__openFileCount is None:
self.__openFileCount = openCount
elif openCount > self.__openFileCount:
runSet.logToDash("WARNING: Possible file leak; open file count" +
" increased from %d to %d" %
(self.__openFileCount, openCount))
self.__openFileCount = openCount
def logTo(self, logIP, logPort, liveIP, livePort):
"Send log messages to the specified host and port"
self.__log.openLog(logIP, logPort, liveIP, livePort)
if logIP is None:
logIP = ''
if logPort is None:
logPort = 0
if liveIP is None:
liveIP = ''
if livePort is None:
livePort = 0
self.__client.xmlrpc.logTo(logIP, logPort, liveIP, livePort)
infoStr = self.__client.xmlrpc.getVersionInfo()
self.__log.debug(
("Version info: %(filename)s %(revision)s" +
" %(date)s %(time)s %(author)s %(release)s" + " %(repo_rev)s") %
get_version_info(infoStr))
def main():
"Main program"
ver_info = "%(filename)s %(revision)s %(date)s %(time)s %(author)s " \
"%(release)s %(repo_rev)s" % get_version_info(SVN_ID)
usage = "%prog [options]\nversion: " + ver_info
p = optparse.OptionParser(usage=usage, version=ver_info)
p.add_option("-C", "--cluster-desc", type="string", dest="clusterDesc",
action="store", default=None,
help="Cluster description name")
p.add_option("-c", "--config-name", type="string", dest="configName",
action="store", default=None,
help="REQUIRED: Configuration name")
p.add_option("", "--delete", dest="delete",
action="store_true", default=True,
help="Run rsync's with --delete")
p.add_option("", "--no-delete", dest="delete",
action="store_false", default=True,
help="Run rsync's without --delete")
p.add_option("-l", "--list-configs", dest="doList",
action="store_true", default=False,
help="List available configs")
p.add_option("-n", "--dry-run", dest="dryRun",
action="store_true", default=False,
help="Don't run rsyncs, just print as they would be run" +
" (disables quiet)")
p.add_option("", "--deep-dry-run", dest="deepDryRun",
action="store_true", default=False,
help="Run rsync's with --dry-run (implies verbose and serial)")
p.add_option("-p", "--parallel", dest="doParallel",
action="store_true", default=True,
help="Run rsyncs in parallel (default)")
p.add_option("-q", "--quiet", dest="quiet",
action="store_true", default=False,
help="Run quietly")
p.add_option("-s", "--serial", dest="doSerial",
action="store_true", default=False,
help="Run rsyncs serially (overrides parallel and unsets" +
" timeout)")
p.add_option("-t", "--timeout", type="int", dest="timeout",
action="store", default=300,
help="Number of seconds before rsync is terminated")
p.add_option("-v", "--verbose", dest="verbose",
action="store_true", default=False,
help="Be chatty")
p.add_option("", "--undeploy", dest="undeploy",
action="store_true", default=False,
help="Remove entire ~pdaq/.m2 and ~pdaq/pDAQ_current dirs" +
" on remote nodes - use with caution!")
p.add_option("", "--nice-adj", type="int", dest="niceAdj",
action="store", default=NICE_ADJ_DEFAULT,
help="Set nice adjustment for remote rsyncs" +
" [default=%default]")
p.add_option("-E", "--express", dest="express",
action="store_true", default=EXPRESS_DEFAULT,
help="Express rsyncs, unsets and overrides any/all" +
" nice adjustments")
opt, args = p.parse_args()
## Work through options implications ##
# A deep-dry-run implies verbose and serial
if opt.deepDryRun:
opt.doSerial = True
opt.verbose = True
opt.quiet = False
# Serial overrides parallel and unsets timout
if opt.doSerial:
opt.doParallel = False
opt.timeout = None
# dry-run implies we want to see what is happening
if opt.dryRun: opt.quiet = False
# Map quiet/verbose to a 3-value tracelevel
traceLevel = 0
if opt.quiet: traceLevel = -1
if opt.verbose: traceLevel = 1
if opt.quiet and opt.verbose: traceLevel = 0
# How often to report count of processes waiting to finish
monitorIval = None
if traceLevel >= 0 and opt.timeout:
monitorIval = max(opt.timeout * 0.01, 2)
if opt.doList:
DAQConfig.showList(None, None)
raise SystemExit
if not opt.configName:
print >>sys.stderr, 'No configuration specified'
p.print_help()
raise SystemExit
try:
cdesc = opt.clusterDesc
config = \
DAQConfigParser.getClusterConfiguration(opt.configName, False,
clusterDesc=cdesc)
except XMLFileNotFound:
print >>sys.stderr, 'Configuration "%s" not found' % opt.configName
p.print_help()
raise SystemExit
if traceLevel >= 0:
if config.descName() is None:
print "CLUSTER CONFIG: %s" % config.configName()
else:
print "CONFIG: %s" % config.configName()
print "CLUSTER: %s" % config.descName()
nodeList = config.nodes()
nodeList.sort()
print "NODES:"
for node in nodeList:
print " %s(%s)" % (node.hostName(), node.locName()),
compList = node.components()
compList.sort()
for comp in compList:
print comp.fullName(),
if comp.isHub():
print "[%s]" % getHubType(comp.id()),
print " ",
print
if not opt.dryRun:
config.writeCacheFile()
ver = store_svnversion()
if traceLevel >= 0:
print "VERSION: %s" % ver
parallel = ParallelShell(parallel=opt.doParallel, dryRun=opt.dryRun,
verbose=(traceLevel > 0 or opt.dryRun),
trace=(traceLevel > 0), timeout=opt.timeout)
pdaqDir = replaceHome(os.environ["HOME"], metaDir)
deploy(config, parallel, os.environ["HOME"], pdaqDir, SUBDIRS, opt.delete,
opt.dryRun, opt.deepDryRun, opt.undeploy, traceLevel, monitorIval,
opt.niceAdj, opt.express)
def main():
"Main program"
ver_info = "%(filename)s %(revision)s %(date)s %(time)s %(author)s "\
"%(release)s %(repo_rev)s" % get_version_info(SVN_ID)
usage = "%prog [options]\nversion: " + ver_info
p = optparse.OptionParser(usage=usage, version=ver_info)
p.add_option("-c",
"--config-name",
action="store",
type="string",
dest="configName")
p.add_option("-d",
"--duration-seconds",
action="store",
type="int",
dest="duration")
p.add_option("-f",
"--flasher-run",
action="store",
type="string",
dest="flasherRun")
p.add_option("-n",
"--num-runs",
action="store",
type="int",
dest="numRuns")
p.add_option("-p",
"--remote-port",
action="store",
type="int",
dest="portNum")
p.add_option("-r",
"--remote-node",
action="store",
type="string",
dest="nodeName")
p.add_option("-s",
"--starting-run",
action="store",
type="int",
dest="startRunNum",
help="Run number to start with")
p.add_option("-x",
"--show-status-xml",
action="store_true",
dest="showXML")
p.set_defaults(nodeName="localhost",
numRuns=10000000,
portNum=DAQPort.DAQRUN,
duration=300,
flasherRun=None,
showXML=False,
startRunNum=None,
configName="hub1001sim")
opt, args = p.parse_args()
runFile = join(environ["HOME"], ".last_pdaq_run")
startRunNum = 1
lastRunNum = getLastRunNum(runFile)
if lastRunNum != None: startRunNum = lastRunNum + 1
if opt.startRunNum: startRunNum = opt.startRunNum
if startRunNum < 1:
raise Exception("Starting run number must be > 0, got %s!" %
startRunNum)
# Connect to DAQ run server
daqiface = DAQRunIface(opt.nodeName, opt.portNum)
# Check for valid flasher input file
if opt.flasherRun and not exists(opt.flasherRun):
print "Flasher file '%s' doesn't exist!" % opt.flasherRun
raise SystemExit
# Check for valid confuration name
if not daqiface.isValidConfig(opt.configName):
print "Run configuration %s does not exist or is not valid!" % opt.configName
raise SystemExit
sleeptime = 0.4
xmlIval = 5
state = None
txml = datetime.now()
runNum = startRunNum
startTime = None
lastStateChg = None
thisSubRun = None
subRunSet = None
try:
while True:
tnow = datetime.now()
if state == None: # Get a well-defined state (probably STOPPED)
state = updateStatus(state, daqiface.getState())
if opt.showXML and (not txml
or tnow - txml > timedelta(seconds=xmlIval)):
showXML(daqiface)
txml = tnow
if state == "STOPPED": # Try to start run
if runNum >= startRunNum + opt.numRuns: break
subRunSet = None # Reset state of subruns
print "Starting run %d..." % runNum
setLastRunNum(runFile, runNum)
try:
daqiface.start(runNum, opt.configName)
startTime = datetime.now()
runNum += 1
state = updateStatus(state, daqiface.getState())
except Exception, e:
print "Failed transition: %s" % e
state = "ERROR"
if state == "STARTING" or state == "RECOVERING" or state == "STOPPING":
time.sleep(1)
state = updateStatus(state, daqiface.getState())
if state == "RUNNING":
doStop = False
if not startTime or tnow - startTime > timedelta(
seconds=opt.duration):
doStop = True
if opt.flasherRun:
if lastStateChg == None: lastStateChg = tnow
# Prep subruns
if subRunSet == None:
subRunSet = SubRunSet(opt.flasherRun)
thisSubRun = subRunSet.next()
if thisSubRun.type == SubRun.FLASH:
print str(thisSubRun.flasherDictList())
status = daqiface.flasher(
thisSubRun.id, thisSubRun.flasherDictList())
if status == 0:
print "WARNING: flasher op failed, check pDAQ logs!"
else:
pass # Don't explicitly send signal if first transition
# is a delay
# Handle transitions
dt = tnow - lastStateChg
if dt > timedelta(seconds=thisSubRun.duration):
print "-- subrun state change --"
thisSubRun = subRunSet.next()
if thisSubRun == None:
doStop = True
elif thisSubRun.type == SubRun.FLASH:
print str(thisSubRun.flasherDictList())
status = daqiface.flasher(
thisSubRun.id, thisSubRun.flasherDictList())
if status == 0:
print "WARNING: flasher op failed, check pDAQ logs!"
else:
status = daqiface.flasher(thisSubRun.id, [])
if status == 0:
print "WARNING: flasher op failed, check pDAQ logs!"
lastStateChg = tnow
if doStop:
try:
daqiface.stop()
state = updateStatus(state, daqiface.getState())
continue
except Exception, e:
print "Failed transition: %s" % e
state = "ERROR"
time.sleep(1)
time.sleep(sleeptime)
state = updateStatus(state, daqiface.getState())
if state == "ERROR":
try:
daqiface.recover()
state = updateStatus(state, daqiface.getState())
except Exception, e:
print "Failed transition: %s" % e
raise SystemExit
print indent + prevState
numList.append(c[2])
dumpComp(prevComp, numList, indent)
def listVerbose(compList, indent=''):
compList.sort(cmpComp)
for c in compList:
print '%s #%d %s#%d at %s:%d M#%d %s' % \
(indent, c[0], c[1], c[2], c[3], c[4], c[5], c[6])
if __name__ == "__main__":
ver_info = "%(filename)s %(revision)s %(date)s %(time)s %(author)s " \
"%(release)s %(repo_rev)s" % get_version_info(SVN_ID)
usage = "%prog [options]\nversion: " + ver_info
p = optparse.OptionParser(usage=usage, version=ver_info)
p.add_option("-v", "--verbose", action="store_true", dest="verbose")
p.set_defaults(verbose=False)
opt, args = p.parse_args()
cncrpc = RPCClient("localhost", DAQPort.CNCSERVER)
try:
nc = cncrpc.rpc_get_num_components()
lc = cncrpc.rpc_list_components()
ns = int(cncrpc.rpc_num_sets())
ids = cncrpc.rpc_runset_listIDs()
def __init__(self,
name="GenericServer",
clusterDesc=None,
copyDir=None,
dashDir=None,
defaultLogDir=None,
runConfigDir=None,
spadeDir=None,
logIP=None,
logPort=None,
liveIP=None,
livePort=None,
restartOnError=True,
forceRestart=True,
testOnly=False,
quiet=False,
defaultRunsetDebug=None):
"Create a DAQ command and configuration server"
self.__name = name
self.__versionInfo = get_version_info(SVN_ID)
self.__id = int(time.time())
self.__clusterDesc = clusterDesc
self.__copyDir = copyDir
self.__dashDir = os.path.join(metaDir, "dash")
self.__runConfigDir = runConfigDir
self.__spadeDir = spadeDir
self.__defaultLogDir = defaultLogDir
self.__restartOnError = restartOnError
self.__forceRestart = forceRestart
self.__quiet = quiet
self.__monitoring = False
self.__live = None
self.__openFileCount = None
super(CnCServer, self).__init__(defaultDebugBits=defaultRunsetDebug)
# close and exit on ctrl-C
#
signal.signal(signal.SIGINT, self.__closeOnSIGINT)
self.__log = self.createCnCLogger(quiet=(testOnly or quiet))
self.__logServer = \
self.openLogServer(DAQPort.CATCHALL, self.__defaultLogDir)
self.__logServer.startServing()
if logIP is None or logPort is None:
logIP = "localhost"
logPort = DAQPort.CATCHALL
self.__log.openLog(logIP, logPort, liveIP, livePort)
if testOnly:
self.__server = None
else:
while True:
try:
# CnCServer needs to be made thread-safe
# before we can thread the XML-RPC server
#
self.__server = ThreadedRPCServer(DAQPort.CNCSERVER)
#self.__server = RPCServer(DAQPort.CNCSERVER)
break
except socket.error, e:
self.__log.error("Couldn't create server socket: %s" % e)
sys.exit("Couldn't create server socket: %s" % e)
quiet = True
killWith9 = False
doKill(doCnC, dryRun, dashDir, verbose, quiet, clusterConfig, killWith9,
parallel)
doLaunch(doCnC, dryRun, verbose, quiet, clusterConfig, dashDir, configDir,
logDir, spadeDir, copyDir, logPort, livePort,
eventCheck=eventCheck, checkExists=checkExists,
startMissing=startMissing, parallel=parallel)
if __name__ == "__main__":
LOGMODE_OLD = 1
LOGMODE_LIVE = 2
LOGMODE_BOTH = LOGMODE_OLD | LOGMODE_LIVE
ver_info = "%(filename)s %(revision)s %(date)s %(time)s %(author)s %(release)s %(repo_rev)s" % get_version_info(SVN_ID)
usage = "%prog [options]\nversion: " + ver_info
p = optparse.OptionParser(usage=usage, version=ver_info)
p.add_option("-C", "--cluster-desc", type="string", dest="clusterDesc",
action="store", default=None,
help="Cluster description name.")
p.add_option("-c", "--config-name", type="string", dest="clusterConfigName",
action="store", default=None,
help="Cluster configuration name, subset of deployed" +
" configuration.")
p.add_option("-e", "--event-check", dest="eventCheck",
action="store_true", default=False,
help="Event builder will validate events")
p.add_option("-f", "--force", dest="force",
action="store_true", default=False,
def main():
"Main program"
ver_info = "%(filename)s %(revision)s %(date)s %(time)s %(author)s " \
"%(release)s %(repo_rev)s" % get_version_info(SVN_ID)
usage = "%prog [options]\nversion: " + ver_info
p = optparse.OptionParser(usage=usage, version=ver_info)
p.add_option("-c", "--config-name", action="store", type="string", dest="configName",
help="REQUIRED: Configuration name")
p.add_option("", "--delete", action="store_true", dest="delete",
help="Run rsync's with --delete")
p.add_option("", "--no-delete", action="store_false", dest="delete",
help="Run rsync's without --delete")
p.add_option("-l", "--list-configs", action="store_true", dest="doList",
help="List available configs")
p.add_option("-n", "--dry-run", action="store_true", dest="dryRun",
help="Don't run rsyncs, just print as they would be run (disables quiet)")
p.add_option("", "--deep-dry-run", action="store_true", dest="deepDryRun",
help="Run rsync's with --dry-run (implies verbose and serial)")
p.add_option("-p", "--parallel", action="store_true", dest="doParallel",
help="Run rsyncs in parallel (default)")
p.add_option("-q", "--quiet", action="store_true", dest="quiet",
help="Run quietly")
p.add_option("-s", "--serial", action="store_true", dest="doSerial",
help="Run rsyncs serially (overrides parallel)")
p.add_option("-t", "--timeout", action="store", type="int", dest="timeout",
help="Number of seconds before rsync is terminated")
p.add_option("-v", "--verbose", action="store_true", dest="verbose",
help="Be chatty")
p.add_option("", "--undeploy", action="store_true", dest="undeploy",
help="Remove entire ~pdaq/.m2 and ~pdaq/pDAQ_current dirs on remote nodes - use with caution!")
p.set_defaults(configName = None,
doParallel = True,
doSerial = False,
verbose = False,
quiet = False,
delete = True,
dryRun = False,
undeploy = False,
deepDryRun = False,
timeout = 60)
opt, args = p.parse_args()
## Work through options implications ##
# A deep-dry-run implies verbose and serial
if opt.deepDryRun:
opt.doSerial = True
opt.verbose = True
opt.quiet = False
# Serial overrides parallel
if opt.doSerial: opt.doParallel = False
# dry-run implies we want to see what is happening
if opt.dryRun: opt.quiet = False
# Map quiet/verbose to a 3-value tracelevel
traceLevel = 0
if opt.quiet: traceLevel = -1
if opt.verbose: traceLevel = 1
if opt.quiet and opt.verbose: traceLevel = 0
rsyncCmdStub = "nice rsync -azLC%s%s" % (opt.delete and ' --delete' or '',
opt.deepDryRun and ' --dry-run' or '')
# The 'SRC' arg for the rsync command. The sh "{}" syntax is used
# here so that only one rsync is required for each node. (Running
# multiple rsync's in parallel appeared to give rise to race
# conditions and errors.)
rsyncDeploySrc = abspath(join(metaDir, '{target,cluster-config,config,dash,src}'))
targetDir = abspath(join(metaDir, 'target'))
try:
config = ClusterConfig(metaDir, opt.configName, opt.doList, False)
except ConfigNotSpecifiedException:
print >>sys.stderr, 'No configuration specified'
p.print_help()
raise SystemExit
if traceLevel >= 0:
print "CONFIG: %s" % config.configName
print "NODES:"
for node in config.nodes:
print " %s(%s)" % (node.hostName, node.locName),
for comp in node.comps:
print "%s:%d" % (comp.compName, comp.compID),
if comp.compName == "StringHub":
print "[%s]" % getHubType(comp.compID)
print " ",
print
if not opt.dryRun:
config.writeCacheFile()
ver = store_svnversion()
if traceLevel >= 0:
print "VERSION: %s" % ver
m2 = join(environ["HOME"], '.m2')
parallel = ParallelShell(parallel=opt.doParallel, dryRun=opt.dryRun,
verbose=(traceLevel > 0 or opt.dryRun),
trace=(traceLevel > 0), timeout=opt.timeout)
done = False
rsyncNodes = getUniqueHostNames(config)
for nodeName in rsyncNodes:
# Check if targetDir (the result of a build) is present
if not opt.undeploy and not isdir(targetDir):
print >>sys.stderr, "ERROR: Target dir (%s) does not exist." % (targetDir)
print >>sys.stderr, "ERROR: Did you run 'mvn clean install assembly:assembly'?"
raise SystemExit
# Ignore localhost - already "deployed"
if nodeName == "localhost": continue
if not done and traceLevel >= 0:
print "COMMANDS:"
done = True
if opt.undeploy:
cmd = 'ssh %s "\\rm -rf %s %s"' % (nodeName, m2, metaDir)
parallel.add(cmd)
continue
rsynccmd = "%s %s %s:%s" % (rsyncCmdStub, rsyncDeploySrc, nodeName, metaDir)
if traceLevel >= 0: print " "+rsynccmd
parallel.add(rsynccmd)
parallel.start()
if opt.doParallel:
parallel.wait()
if traceLevel <= 0 and not opt.dryRun:
needSeparator = True
rtnCodes = parallel.getReturnCodes()
for i in range(len(rtnCodes)):
result = parallel.getResult(i)
if rtnCodes[i] != 0 or len(result) > 0:
if needSeparator:
print "----------------------------------"
needSeparator = False
print "\"%s\" returned %s:\n%s" % \
(parallel.getCommand(i), str(rtnCodes[i]), result)
def main():
"Main program"
ver_info = "%(filename)s %(revision)s %(date)s %(time)s %(author)s "\
"%(release)s %(repo_rev)s" % get_version_info(SVN_ID)
usage = "%prog [options]\nversion: " + ver_info
p = optparse.OptionParser(usage=usage, version=ver_info)
p.add_option("-c",
"--config-name",
type="string",
dest="runConfig",
action="store",
default=None,
help="Run configuration name")
p.add_option("-d",
"--duration-seconds",
type="string",
dest="duration",
action="store",
default="300",
help="Run duration (in seconds)")
p.add_option("-f",
"--flasher-run",
type="string",
dest="flasherRun",
action="store",
default=None,
help="Name of flasher run configuration file")
p.add_option("-n",
"--num-runs",
type="int",
dest="numRuns",
action="store",
default=10000000,
help="Number of runs")
p.add_option("-r",
"--remote-host",
type="string",
dest="remoteHost",
action="store",
default="localhost",
help="Name of host on which CnCServer is running")
p.add_option("-s",
"--showCommands",
dest="showCmd",
action="store_true",
default=False,
help="Show the commands used to deploy and/or run")
p.add_option("-x",
"--showCommandOutput",
dest="showCmdOut",
action="store_true",
default=False,
help="Show the output of the deploy and/or run commands")
opt, args = p.parse_args()
cnc = CnCRun(showCmd=opt.showCmd, showCmdOutput=opt.showCmdOut)
clusterCfg = cnc.getActiveClusterConfig()
if clusterCfg is None:
raise SystemExit("Cannot determine cluster configuration")
duration = getDurationFromString(opt.duration)
for r in range(opt.numRuns):
run = cnc.createRun(None, opt.runConfig, flashName=opt.flasherRun)
if opt.flasherRun is None:
run.start(duration)
else:
#run.start(duration, flashTimes, flashPause, False)
raise SystemExit("flasher runs with ExpControSkel not implemented")
try:
try:
run.wait()
except KeyboardInterrupt:
print "Run interrupted by user"
break
finally:
print >> sys.stderr, "Stopping run..."
run.finish()