def main():
'''
Process:
- Parse the CLI command into g.various data items.
- Validate user-supplied directories; get environment variables.
- Make queues to send instructions to pool of worker processes.
- Create pool of worker processes.
- Query the searchspace for the stream of instructions
- For each instruction from database selection, get dict for line
- Using dict args, construct plausible command lines, into file
- For each instruction, expand to the number of samples (seeds) to use.
- When we finally have a single instruction to execute, queue that
to the worker jobs.
- When all instructions have been queued, close down the worker processes.
'''
NTRC.ntracef(0, "MAIN", "Begin.")
NTRC.ntracef(0, "MAIN", "TRACE traceproduction|%s|" % NTRC.isProduction())
sBrokerCommand = fnsReconstituteCommand(sys.argv)
fnbMaybeLogCommand(sBrokerCommand)
NTRC.ntracef(0, "MAIN", "command=|%s|" % (sBrokerCommand.rstrip()))
# Get args from CLI and put them into the global data
dCliDict = brokercli.fndCliParse("")
# Carefully insert any new CLI values into the Global object.
dCliDictClean = {
k: util.fnIntPlease(v)
for k, v in dCliDict.items() if v is not None
}
g.__dict__.update(dCliDictClean)
# Validate that the user-specified directories exist.
if not fnbValidateDir(g.sFamilyDir):
raise ValueError("FamilyDir \"%s\" not found" % (g.sFamilyDir))
if not fnbValidateDir("%s/%s" % (g.sFamilyDir, g.sSpecificDir)):
raise ValueError("SpecificDir \"%s\" not found" % (g.sSpecificDir))
# Get command templates from external file.
fnGetCommandTemplates(g.sCommandListFilename)
# Construct database query for this invocation.
g.cFmt = brokerformat.CFormat()
dQuery = g.cFmt.fndFormatQuery(dCliDict, g)
# Look for overriding environment variables
fnvGetEnvironmentOverrides()
# Open the database to keep "done" records,
# and delete moldy, old in-progress records.
g.mdb = searchdatabasemongo.CSearchDatabase(
g.sSearchDbMongoName, g.sSearchDbProgressCollectionName,
g.sSearchDbDoneCollectionName)
g.mdb.fnvDeleteProgressCollection()
# Get the set of instructions for today from database.
NTRC.tracef(
0, "MAIN",
"proc querydict2|%s|" % (list(util.fngSortDictItemsByKeys(dQuery))))
itAllInstructions = searchspace.fndgGetSearchSpace(g.sInsDir, g.sInsTyp,
dQuery)
# Start the start-end threads.
# Define queues.
# Need a Multiprocessing Manager to own the output queue. (Do we?) (Yes.)
mpmgr = mp.Manager()
g.qJobs = mp.Queue()
g.qOutput = mpmgr.Queue()
# Start pool of worker processes.
g.cWorkersInst = cworkers.CWorkers(nservers=g.nCores,
qinputjobs=g.qJobs,
qoutputdata=g.qOutput)
# If this wasn't just a listonly run, do all the cases.
if not g.sListOnly.startswith("Y"):
NTRC.ntracef(3, "MAIN", "proc all instr|%s|" % (g.lGiantInstr))
else:
NTRC.ntracef(0, "MAIN", "Listonly.")
nRuns = fnnProcessAllInstructions(itAllInstructions)
NTRC.ntracef(0, "MAIN", "End queued all runs ncases|%s|" % (g.nCases, ))
def dumpParamsIntoLog():
# We want a log file to be self-contained, so record all sorts
# of information in it about the parameters that resulted in
# the answers.
lg.logInfo("MAIN","Simulation parameters")
lg.logInfo("MAIN","Command line|%s|" % (sys.argv))
sCmd = fnsReconstituteCommand(sys.argv)
lg.logInfo("MAIN","Usable CLI line|%s|" % (sCmd))
lg.logInfo("PARAMS","familydir|%s| specificdir|%s|"
% (G.sFamilyDir,G.sSpecificDir))
lg.logInfo("PARAMS","RANDOM random seed|%d| "
% (G.nRandomSeed))
lg.logInfo("PARAMS","begin simulation timelimit|%d|hr=|%d|metricyr "
"defaultlimit|%d|hr=|%d|metricyr"
% (G.nSimLength, G.nSimLength/10000,
G.nSimLengthDefault, G.nSimLengthDefault/10000))
G.nPoliteTimer = int(os.getenv("NPOLITE", P.nPoliteTimer))
lg.logInfo("PARAMS","POLITE time|%s|msec" % (G.nPoliteTimer))
lg.logInfo("PARAMS","LOG logfile|%s| loglevel|%s|"
% (G.sLogFile,G.sLogLevel))
lg.logInfo("PARAMS", "TRACE traceproduction|%s|" % NTRC.isProduction())
#-----------------------------------------------------------
# C l i e n t params
NTRC.ntracef(3,"MAIN","client params dict|%s|"
% (G.dClientParams))
for sClient in G.dClientParams:
lCollections = G.dClientParams[sClient]
for lCollection in lCollections:
(sCollection,nQuality,nDocs) = lCollection
lg.logInfo("PARAMS","CLIENT client|%s| collection|%s| quality|%d| "
"ndocs|%d|"
% (sClient,sCollection,nQuality,nDocs))
lg.logInfo("PARAMS", "ALLCLIENTS nDocuments|%s| override if nz" % (G.nDocuments))
#-----------------------------------------------------------
# S e r v e r params
NTRC.ntracef(3,"MAIN","server params dict|%s|"
% (G.dServerParams))
for sServer in G.dServerParams:
(nQuality,nShelfSize) = G.dServerParams[sServer][0]
lg.logInfo("PARAMS","SERVER server|%s| quality|%d| shelfsize|%d|TB"
% (sServer,nQuality,nShelfSize))
lg.logInfo("PARAMS","SERVER DefaultHalflife|%s|"
% (G.fServerDefaultHalflife))
#-----------------------------------------------------------
# S h e l f params
NTRC.ntracef(3,"MAIN","shelf params dict|%s|" % (G.dShelfParams))
for nQuality in G.dShelfParams:
(nSmallFailureRate,nShelfFailureRate) = G.dShelfParams[nQuality][0]
lg.logInfo("PARAMS","SHELF quality|%d| smallfailrate|%d|Khr=|%d|yr "
"shelffailrate|%d|Khr=|%d|yr"
% (nQuality, nSmallFailureRate, nSmallFailureRate*1000/8766,
nShelfFailureRate,nShelfFailureRate*1000/8766))
#-----------------------------------------------------------
# D i s t r i b u t i o n policy params.
NTRC.ntracef(3,"MAIN","distn params dict|%s|" % (G.dDistnParams))
for nValue in G.dDistnParams:
(nQuality,nCopies) = G.dDistnParams[nValue][0]
lg.logInfo("PARAMS","DISTRIBUTION value|%d| quality|%d| copies|%d|"
% (nValue,nQuality,nCopies))
#-----------------------------------------------------------
# D o c u m e n t S i z e params.
NTRC.ntracef(3,"MAIN","document params dict|%s|" % (G.dDistnParams))
for nValue in G.dDocParams:
for lMode in G.dDocParams[nValue]:
(nPercent,nMean,nSdev) = lMode
lg.logInfo("PARAMS","DOCUMENT value|%d| percent|%d| "
"meanMB|%d| sd|%d|"
% (nValue,nPercent,nMean,nSdev))
#-----------------------------------------------------------
# A u d i t params.
lg.logInfo("PARAMS","AUDIT interval hours|%s| segments|%s| type|%s| "
"bandwidth Mbps|%s|"
% (G.nAuditCycleInterval, G.nAuditSegments, G.sAuditStrategy,
G.nBandwidthMbps))
#-----------------------------------------------------------
# G l i t c h params.
lg.logInfo("PARAMS","GLITCH freq|%d| impact|%d| span|%d| decay|%d| "
"maxlife|%d| ignorelimit|%.3f|"
% (G.nGlitchFreq, G.nGlitchImpact, G.nGlitchSpan, G.nGlitchDecay,
G.nGlitchMaxlife, G.fGlitchIgnoreLimit))
#-----------------------------------------------------------
# S h o c k params.
lg.logInfo("PARAMS","SHOCKS freq|%d| impact|%d| span|%d| "
"maxlife|%s| type|%s|"
% (G.nShockFreq, G.nShockImpact, G.nShockSpan, G.nShockMaxlife,
G.nShockType ))
def main():
'''
Process:
- Parse the CLI command into g.various data items.
- Validate user-supplied directories; get environment variables.
- Query the searchspace for the stream of instructions
- For each instruction from database selection, get dict for line
- Using dict args, construct plausible command lines, into file
- Check to see that there aren't too many similar processes
already running; if too many, then wait.
- Launch ListActor process to execute commands.
- Wait a polite interval before launching another.
'''
NTRC.ntracef(0, "MAIN", "Begin.")
NTRC.ntracef(0, "MAIN", "TRACE traceproduction|%s|" % NTRC.isProduction())
sBrokerCommand = fnsReconstituteCommand(sys.argv)
fnbMaybeLogCommand(sBrokerCommand)
NTRC.ntracef(0, "MAIN", "command=|%s|" % (sBrokerCommand.rstrip()))
# Get args from CLI and put them into the global data
dCliDict = brokercli.fndCliParse("")
# Carefully insert any new CLI values into the Global object.
dCliDictClean = {k:util.fnIntPlease(v) for k,v in dCliDict.items()
if v is not None}
g.__dict__.update(dCliDictClean)
# Validate that the user-specified directories exist.
if not fnbValidateDir(g.sFamilyDir):
raise ValueError("FamilyDir \"%s\" not found" % (g.sFamilyDir))
if not fnbValidateDir("%s/%s" % (g.sFamilyDir, g.sSpecificDir)):
raise ValueError("SpecificDir \"%s\" not found" % (g.sSpecificDir))
# Get command templates from external file.
fnGetCommandTemplates(g.sCommandListFilename)
# Construct database query for this invocation.
g.cFmt = brokerformat.CFormat()
dQuery = g.cFmt.fndFormatQuery(dCliDict, g)
# Look for overriding environment variables
fnvGetEnvironmentOverrides()
# Open the database to keep "done" records,
# and delete moldy, old in-progress records.
g.mdb = searchdatabasemongo.CSearchDatabase(g.sSearchDbMongoName,
g.sSearchDbProgressCollectionName,
g.sSearchDbDoneCollectionName)
g.mdb.fnvDeleteProgressCollection()
# Get the set of instructions for today from database.
NTRC.tracef(0,"MAIN","proc querydict2|%s|" % ((dQuery)))
itAllInstructions = searchspace.fndgGetSearchSpace(g.sInsDir, g.sInsTyp,
dQuery)
nRuns = fnnProcessAllInstructions(itAllInstructions)
# If this wasn't just a listonly run, do all the cases.
if not g.sListOnly.startswith("Y"):
NTRC.ntracef(3, "MAIN", "proc all instr|%s|" % (g.lGiantInstr))
nCases = nb.fntRunEverything(g, iter(g.lGiantInstr)
, g.nCoreTimer, g.nStuckLimit)
else:
nCases = len(g.lGiantInstr)
NTRC.ntracef(0, "MAIN", "End ncases|%s|" % (nCases,))
def main():
'''
Process:
- Parse the CLI command into g.various data items.
- Validate user-supplied directories; get environment variables.
- Query the searchspace for the stream of instructions
- For each instruction from database selection, get dict for line
- Using dict args, construct plausible command lines, into file
- Check to see that there aren't too many similar processes
already running; if too many, then wait.
- Launch ListActor process to execute commands.
- Wait a polite interval before launching another.
'''
NTRC.ntracef(0, "MAIN", "Begin.")
NTRC.ntracef(0, "MAIN", "TRACE traceproduction|%s|" % NTRC.isProduction())
def fnbQEnd():
return g.bLast
sBrokerCommand = fnsReconstituteCommand(sys.argv)
fnbMaybeLogCommand(sBrokerCommand)
NTRC.ntracef(0, "MAIN", "command=|%s|" % (sBrokerCommand.rstrip()))
# Get args from CLI and put them into the global data
dCliDict = brokercli.fndCliParse("")
# Carefully insert any new CLI values into the Global object.
dCliDictClean = {
k: util.fnIntPlease(v)
for k, v in dCliDict.items() if v is not None
}
g.__dict__.update(dCliDictClean)
# Validate that the user-specified directories exist.
if not fnbValidateDir(g.sFamilyDir):
raise ValueError("FamilyDir \"%s\" not found" % (g.sFamilyDir))
if not fnbValidateDir("%s/%s" % (g.sFamilyDir, g.sSpecificDir)):
raise ValueError("SpecificDir \"%s\" not found" % (g.sSpecificDir))
# Get command templates from external file.
fnGetCommandTemplates(g.sCommandListFilename)
# Construct database query for this invocation.
g.cFmt = brokerformat.CFormat()
dQuery = g.cFmt.fndFormatQuery(dCliDict, g)
# Look for overriding environment variables
fnvGetEnvironmentOverrides()
# Open the database to keep "done" records,
# and delete moldy, old in-progress records.
g.mdb = searchdatabasemongo.CSearchDatabase(
g.sSearchDbMongoName, g.sSearchDbProgressCollectionName,
g.sSearchDbDoneCollectionName)
g.mdb.fnvDeleteProgressCollection()
# Get the set of instructions for today from database.
NTRC.tracef(
0, "MAIN",
"proc querydict2|%s|" % (list(util.fngSortDictItemsByKeys(dQuery))))
itAllInstructions = searchspace.fndgGetSearchSpace(g.sInsDir, g.sInsTyp,
dQuery)
# Start the start-end threads.
nb.fntRunEverything(g, g.qInstructions, fnbQEnd, g.nCoreTimer,
g.nStuckLimit)
# If this wasn't just a listonly run, do all the cases.
if not g.sListOnly.startswith("Y"):
NTRC.ntracef(3, "MAIN", "proc all instr|%s|" % (g.lGiantInstr))
else:
NTRC.ntracef(0, "MAIN", "Listonly.")
nRuns = fnnProcessAllInstructions(itAllInstructions)
NTRC.ntracef(0, "MAIN", "End queued all runs ncases|%s|" % (g.nCases, ))