def msGentlyFormat(self, mysCmd, mydVals, myg, myCG):
'''
Like string.format() but does not raise exception if the string
contains a name request for which the dictionary does not have
a value. Leaves unfulfilled name requests in place.
Method: construct a dictionary that contains something for every
name requested in the string. The value is either a supplied
value from the caller or a placeholder for the name request.
Then use the now-defanged string.format() method.
This is way harder than it ought to be, grumble.
'''
# Make a dictionary from the names requested in the string
# that just replaces the request '{foo}' with itself.
sReNames = '(:?\{([^\}]+)\})+'
oReNames = re.compile(sReNames)
lNameTuples = oReNames.findall(mysCmd)
NTRC.ntracef(3, "FMT", "proc gently tuples|%s|" % (lNameTuples))
lNames = [x[1] for x in lNameTuples]
dNames = dict(zip(lNames, map(lambda s: "{" + s + "}", lNames)))
# Pick up any specified values in the global object
# and from CLI args.
dNames.update(dict(vars(myCG)))
dNames.update(dict(vars(myg)))
# And then add values from the specific instructions.
dNames.update(mydVals)
NTRC.ntrace(3, "proc gently dnames|%s|" % (dNames))
sOut = mysCmd.format(**dNames)
return sOut
def msGentlyFormat(self, mysCmd, mydVals, myg, myCG):
'''
Like string.format() but does not raise exception if the string
contains a name request for which the dictionary does not have
a value. Leaves unfulfilled name requests in place.
Method: construct a dictionary that contains something for every
name requested in the string. The value is either a supplied
value from the caller or a placeholder for the name request.
Then use the now-defanged string.format() method.
This is way harder than it ought to be, grumble.
'''
# Make a dictionary from the names requested in the string
# that just replaces the request '{foo}' with itself.
sReNames = '(:?\{([^\}]+)\})+'
oReNames = re.compile(sReNames)
lNameTuples = oReNames.findall(mysCmd)
NTRC.ntracef(3,"FMT","proc gently tuples|%s|" % (lNameTuples))
lNames = [x[1] for x in lNameTuples]
dNames = dict(zip(lNames, map(lambda s: "{"+s+"}", lNames)))
# Pick up any specified values in the global object
# and from CLI args.
dNames.update(dict(vars(myCG)))
dNames.update(dict(vars(myg)))
# And then add values from the specific instructions.
dNames.update(mydVals)
NTRC.ntrace(3,"proc gently dnames|%s|" % (dNames))
sOut = mysCmd.format(**dNames)
return sOut
def __init__(self, name, life):
self.ID = name
self.life = life
self._timer = rt.CResettableTimer(G.env, life, shockcall, shockinter, self.ID)
NTRC.ntrace(0, "proc shock.init before waitfor t|%s|" % G.env.now)
self._timer.start()
G.env.process(self.waitforshock())
NTRC.ntrace(0, "proc shock.init after waitfor t|%s|" % G.env.now)
def main(mysDbName, mysCollectionName):
oDb = fnoOpenDb(mysDbName)
betterbezero = fniClearCollection(oDb, mysCollectionName)
sFilename = fnsGetFilename()
oCollection = oDb[mysCollectionName]
nRecordCount = fniPutFileToDb(sFilename, " ", oCollection)
dTmp = oCollection.find_one()
NTRC.ntrace(0,"======\n{}\n======".format(dTmp))
NTRC.ntrace(0,"nRecs stored|{}|".format(nRecordCount))
def fniPutFileToDb(mysFilename, mysSeparator, myoCollection):
'''
Open file to iterable, then use that to add file to db.
'''
# Store fresh, new records into the collection.
with open(mysFilename, "r") as fhInfile:
NTRC.ntrace(3, "File {} opened.".format(mysFilename))
nRec = fnnPutIterToDb(fhInfile,mysSeparator,myoCollection)
return nRec
def main():
NTRC.ntrace(0, "Begin.")
# Get args from CLI and put them into the global data
dCliDict = fndCliParse("")
# Carefully insert any new CLI values into the Global object.
dCliDictClean = {k: v for k, v in dCliDict.items() if v is not None}
g.__dict__.update(dCliDictClean)
# Use naked Mongo functions not suitable for searchdatabasemongo library.
# Since MongoDB is a system-wide singleton resource, there is no need
# to get any name arguments for this command.
client = pymongo.MongoClient()
client.drop_database(g.sDatabaseName)
NTRC.ntrace(0, "End.")
def main():
NTRC.ntrace(0,"Begin.")
# Get args from CLI and put them into the global data
dCliDict = fndCliParse("")
# Carefully insert any new CLI values into the Global object.
dCliDictClean = {k:v for k,v in dCliDict.items() if v is not None}
g.__dict__.update(dCliDictClean)
# Use naked Mongo functions not suitable for searchdatabasemongo library.
# Since MongoDB is a system-wide singleton resource, there is no need
# to get any name arguments for this command.
client = pymongo.MongoClient()
client.drop_database(g.sDatabaseName)
NTRC.ntrace(0,"End.")
def m_CreateWorkers(self):
''' Create worker pool and start them all.
The number of workers was specified in the class construction.
'''
lProcs = []
for iProcessNum in range(self.nParallel):
proc = mp.Process(target=doManyJobs,
args=(self.qJobs, ),
name=f'CWorker-{iProcessNum+1}')
lProcs.append(proc)
proc.start()
NTRC.ntrace(3, "proc worker|%s| started on q|%s|" % (proc, self.qJobs))
self.lprocWorkers = lProcs # Save a list of all workers.
return lProcs
def mainsim_setup_post():
# C O L L E C T D A T A
sFamilyDir = bottle.request.forms.get("sFamilyDir")
sSpecificDir = bottle.request.forms.get("sSpecificDir")
bClearDirs = bottle.request.forms.get("bClearDirs")
bClearDone = bottle.request.forms.get("bClearDone")
sAction = bottle.request.forms.get("submit")
sOK = bottle.request.POST.ok
sCancel = bottle.request.POST.cancel
msg = "mainsim_setup_post: done"
# F O R M D I C T I O N A R Y O F S U B S T I T U T I O N S
# Make a dictionary to use to substitute params into CLI command.
dVals = dict(sFamilyDir=sFamilyDir
,sSpecificDir=sSpecificDir
,bClearDirs=("Yes" if bClearDirs else "No")
,bClearDone=("Yes" if bClearDone else "No")
,sOK=sOK
,sCancel=sCancel
,sAction=("DONE" if sOK else "CANCELLED")
,msg=msg
)
NTRC.ntrace(3,"proc first dict|%s|" % (dVals))
if sOK:
# If instructed to clear area, do that first.
if bClearDirs:
sClearDirsCmd = cCmd.mMakeCmd(sCmdClearDirs, dVals)
lCmdOut = cCmd.mDoCmdLst(sClearDirsCmd)
dVals["sResultClearDirs"] = fnsMakeReadable(sClearDirsCmd, lCmdOut)
# Use standard script to setup output dirs.
sSetupDirsCmd = cCmd.mMakeCmd(sCmdSetupDirs, dVals)
lCmdOut = cCmd.mDoCmdLst(sSetupDirsCmd)
dVals["sResultSetupDirs"] = fnsMakeReadable(sSetupDirsCmd, lCmdOut)
# Use standard script to clear done records.
if bClearDone:
sClearDoneCmd = cCmd.mMakeCmd(sCmdClearDone, dVals)
lCmdOut = cCmd.mDoCmdLst(sClearDoneCmd)
dVals["sResultClearDone"] = fnsMakeReadable(sClearDoneCmd, lCmdOut)
# Tell user what we did.
lVals = ["k:|%s| v:|%s|" % (k, v) for (k, v) in sorted(dVals.items())]
sVals = "\n<br/>".join(lVals)
sOut = sVals
return dVals
def main():
NTRC.ntrace(0, "Begin.")
# Get args from CLI and put them into the global data
dCliDict = fndCliParse("")
# Carefully insert any new CLI values into the Global object.
dCliDictClean = {k: v for k, v in dCliDict.items() if v is not None}
g.__dict__.update(dCliDictClean)
# Since we're deleting an arbitrary collection from the db,
# it doesn't matter if we pretend that it is a specific one
# with a different name today.
g.mdb = searchdatabasemongo.CSearchDatabase(g.sDatabaseName)
lNames = g.mdb.fnlGetCollections()
for sName in lNames:
print sName
NTRC.ntrace(0, "End.")
def main():
NTRC.ntrace(0,"Begin.")
# Get args from CLI and put them into the global data
dCliDict = fndCliParse("")
# Carefully insert any new CLI values into the Global object.
dCliDictClean = {k:v for k,v in dCliDict.items() if v is not None}
g.__dict__.update(dCliDictClean)
# Since we're deleting an arbitrary collection from the db,
# it doesn't matter if we pretend that it is a specific one
# with a different name today.
g.mdb = searchdatabasemongo.CSearchDatabase(g.sDatabaseName,
g.sCollectionName, g.sCollectionName)
g.mdb.fnvDeleteProgressCollection()
NTRC.ntrace(0,"Cleared collection|{1}| in database|{0}|".format(g.sDatabaseName,g.sCollectionName))
def waitforshock(self):
NTRC.ntrace(0, "proc shock.waitfor before yield t|%s|" % G.env.now)
yield self._timer.event
NTRC.ntrace(0, "proc shock.waitfor after yield t|%s|" % G.env.now)
NTRC.ntrace(0, "proc shock.waitfor reset the server timer here!")
G.sa.timer.stop()
G.sa.timer.setdelay(33333).start()
NTRC.ntrace(0, "proc shock.waitfor done reset server timer")
def mainsim_setup_post():
# C O L L E C T D A T A
sFamilyDir = bottle.request.forms.get("sFamilyDir")
sSpecificDir = bottle.request.forms.get("sSpecificDir")
bClearDirs = bottle.request.forms.get("bClearDirs")
bClearDone = bottle.request.forms.get("bClearDone")
sAction = bottle.request.forms.get("submit")
sOK = bottle.request.POST.ok
sCancel = bottle.request.POST.cancel
msg = "mainsim_setup_post: done"
# F O R M D I C T I O N A R Y O F S U B S T I T U T I O N S
# Make a dictionary to use to substitute params into CLI command.
dVals = dict(sFamilyDir=sFamilyDir,
sSpecificDir=sSpecificDir,
bClearDirs=("Yes" if bClearDirs else "No"),
bClearDone=("Yes" if bClearDone else "No"),
sOK=sOK,
sCancel=sCancel,
sAction=("DONE" if sOK else "CANCELLED"),
msg=msg)
NTRC.ntrace(3, "proc first dict|%s|" % (dVals))
if sOK:
# If instructed to clear area, do that first.
if bClearDirs:
sClearDirsCmd = cCmd.mMakeCmd(sCmdClearDirs, dVals)
lCmdOut = cCmd.mDoCmdLst(sClearDirsCmd)
dVals["sResultClearDirs"] = fnsMakeReadable(sClearDirsCmd, lCmdOut)
# Use standard script to setup output dirs.
sSetupDirsCmd = cCmd.mMakeCmd(sCmdSetupDirs, dVals)
lCmdOut = cCmd.mDoCmdLst(sSetupDirsCmd)
dVals["sResultSetupDirs"] = fnsMakeReadable(sSetupDirsCmd, lCmdOut)
# Use standard script to clear done records.
if bClearDone:
sClearDoneCmd = cCmd.mMakeCmd(sCmdClearDone, dVals)
lCmdOut = cCmd.mDoCmdLst(sClearDoneCmd)
dVals["sResultClearDone"] = fnsMakeReadable(sClearDoneCmd, lCmdOut)
# Tell user what we did.
lVals = ["k:|%s| v:|%s|" % (k, v) for (k, v) in sorted(dVals.items())]
sVals = "\n<br/>".join(lVals)
sOut = sVals
return dVals
def ageserver(self,life):
NTRC.ntrace(0, "proc server.age before yield id|%s| t|%s|"
% (self.ID, G.env.now))
result = None
while not result or result == "CANCELED":
NTRC.ntrace(3,"proc server.age |%s| waiting for event |%s|"
% (self, self._timer.event))
result = yield self._timer.event
NTRC.ntrace(0, "proc server.age after yield id|%s| t|%s| result|%s|"
% (self.ID, G.env.now, result))
NTRC.ntrace(0, "proc server.age end of yield id|%s| t|%s| result|%s|"
% (self.ID, G.env.now, result))
def mainNewBroker(gl):
NTRC.ntrace(3, "proc params ncases|%s| nparallel|%s| "
"nwaitmsec|%s| nwaitmany|%s|"
% (gl.nCases, gl.nParallel, gl.nWaitMsec, gl.nWaitHowMany))
# Main loop
# Create list of instructions. Each instruction is a list of
# command strings.
lLinesTemp = [sLine.lstrip()
for sLine in sTempListOfCommands.split('\n')
]
# And make a list of instructions for each case.
llsInstructionsTemp = [lLinesTemp] * gl.nCases
tTmp = fntRunEverything(gl, iter(llsInstructionsTemp)
, gl.nWaitMsec, gl.nWaitHowMany)
(gl.nWaitedForSlot, gl.nWaitedForDone) = (tTmp.slot, tTmp.done)
return [] # used to be llOut in early proto
def mainNewBroker(gl):
NTRC.ntrace(3, "proc params ncases|%s| nparallel|%s| "
"nwaitmsec|%s| nwaitmany|%s|"
% (gl.nCases, gl.nParallel, gl.nWaitMsec, gl.nWaitHowMany))
# Main loop
# Create list of instructions. Each instruction is a list of
# command strings.
lLinesTemp = [sLine.lstrip()
for sLine in sTempListOfCommands.split('\n')
]
# And make a list of instructions for each case.
llsInstructionsTemp = [lLinesTemp] * gl.nCases
tTmp = fntRunEverything(gl, iter(llsInstructionsTemp)
, gl.nWaitMsec, gl.nWaitHowMany)
(gl.nWaitedForSlot, gl.nWaitedForDone) = (tTmp.slot, tTmp.done)
return [] # used to be llOut in early proto
def main():
NTRC.ntrace(0,"Begin.")
# Get args from CLI and put them into the global data
dCliDict = fndCliParse("")
# Carefully insert any new CLI values into the Global object.
dCliDictClean = {k:v for k,v in dCliDict.items() if v is not None}
g.__dict__.update(dCliDictClean)
# Since we're deleting an arbitrary collection from the db,
# it doesn't matter if we pretend that it is a specific one
# with a different name today.
g.mdb = searchdatabasemongo.CSearchDatabase(g.sDatabaseName,
g.sCollectionName, g.sCollectionName)
lCollection = g.mdb.fnlGetDoneCollection()
# Just print all the items in collection.
print ("Found %d items:" % (len(lCollection)))
for (nItem, dItem) in enumerate(lCollection):
nItem += 1 # Humans are one-based, not zero-based.
sDoneId = dItem["sDoneId"]
dResults = dItem["info"]
sCopies = dResults["copies"]
sSeed = dResults["seed"]
sLost = dResults["lost"]
sTimestamp = dResults["timestamp"]
sLifem = dResults["lifem"]
sAuditFreq = dResults["auditfrequency"]
sAuditSegs = dResults["auditsegments"]
sShockFreq = dResults["shockfreq"]
sShockImpact = dResults["shockimpact"]
sShockSpan = dResults["shockspan"]
sGlitchFreq = dResults["glitchfreq"]
sGlitchImpact = dResults["glitchimpact"]
#sGlitchSpan = dResults["glitchspan"]
print (
"#%4d cop %2s life %4s lost %4s audit %5s-%1s shock %s %s %s %s %s id|%s|"
% (nItem, sCopies, sLifem, sLost, sAuditFreq, sAuditSegs,
sShockFreq, sShockImpact, sShockSpan,
sSeed, sTimestamp, sDoneId)
)
def main():
NTRC.ntrace(0, "Begin.")
# Get args from CLI and put them into the global data
dCliDict = fndCliParse("")
# Carefully insert any new CLI values into the Global object.
dCliDictClean = {k: v for k, v in dCliDict.items() if v is not None}
g.__dict__.update(dCliDictClean)
# Since we're deleting an arbitrary collection from the db,
# it doesn't matter if we pretend that it is a specific one
# with a different name today.
g.mdb = searchdatabasemongo.CSearchDatabase(g.sDatabaseName,
g.sCollectionName,
g.sCollectionName)
lCollection = g.mdb.fnlGetDoneCollection()
# Just print all the items in collection.
print("Found %d items:" % (len(lCollection)))
for (nItem, dItem) in enumerate(lCollection):
nItem += 1 # Humans are one-based, not zero-based.
sDoneId = dItem["sDoneId"]
dResults = dItem["info"]
sCopies = dResults["copies"]
sSeed = dResults["seed"]
sLost = dResults["lost"]
sTimestamp = dResults["timestamp"]
sLifem = dResults["lifem"]
sAuditFreq = dResults["auditfrequency"]
sAuditSegs = dResults["auditsegments"]
sShockFreq = dResults["shockfreq"]
sShockImpact = dResults["shockimpact"]
sShockSpan = dResults["shockspan"]
sGlitchFreq = dResults["glitchfreq"]
sGlitchImpact = dResults["glitchimpact"]
#sGlitchSpan = dResults["glitchspan"]
print(
"#%4d cop %2s life %4s lost %4s audit %5s-%1s shock %s %s %s %s %s id|%s|"
%
(nItem, sCopies, sLifem, sLost, sAuditFreq, sAuditSegs, sShockFreq,
sShockImpact, sShockSpan, sSeed, sTimestamp, sDoneId))
def main():
NTRC.ntrace(0,"Begin.")
# Get args from CLI and put them into the global data
dCliDict = fndCliParse("")
# Carefully insert any new CLI values into the Global object.
dCliDictClean = {k:v for k,v in dCliDict.items() if v is not None}
g.__dict__.update(dCliDictClean)
# Since we're deleting an arbitrary collection from the db,
# it doesn't matter if we pretend that it is a specific one
# with a different name today.
g.mdb = searchdatabasemongo.CSearchDatabase(g.sDatabaseName,
g.sCollectionName, g.sCollectionName)
lCollection = g.mdb.fnlGetCollection(g.sCollectionName)
# Just print all the items in collection.
nItem = 0
for thing in lCollection:
nItem += 1
print("--- %s ------ collection %s ------ db %s -----------------------------" % (nItem, g.sCollectionName, g.sDatabaseName))
print(thing)
def m_CreateOutputReceiver(self, myfReceiver=None):
''' Make one more async process to receive job outputs.
If there is no output queue at all, then there will be no
receiver process.
If the user specifies an output queue, but does not specify
a function for the receiver, then the default receiver
will be used.
'''
if CWorkers.qOutput:
if not myfReceiver:
# Take default if none specified.
procRcvrToday = defaultReceiveOutput
else:
procRcvrToday = myfReceiver
procReceiver = mp.Process(target=procRcvrToday,
args=(CWorkers.qOutput,))
procReceiver.start()
NTRC.ntrace(3, "proc rcvr|%s||%s| started on q|%s|"
% (procRcvrToday, procReceiver, CWorkers.qOutput))
self.procOutput = procReceiver
else:
self.procOutput = None
return self.procOutput
def main():
NTRC.ntrace(0, "Begin.")
# Get args from CLI and put them into the global data
dCliDict = fndCliParse("")
# Carefully insert any new CLI values into the Global object.
dCliDictClean = {k: v for k, v in dCliDict.items() if v is not None}
g.__dict__.update(dCliDictClean)
# Since we're deleting an arbitrary collection from the db,
# it doesn't matter if we pretend that it is a specific one
# with a different name today.
g.mdb = searchdatabasemongo.CSearchDatabase(g.sDatabaseName,
g.sCollectionName,
g.sCollectionName)
lCollection = g.mdb.fnlGetCollection(g.sCollectionName)
# Just print all the items in collection.
nItem = 0
for thing in lCollection:
nItem += 1
print(
"--- %s ------ collection %s ------ db %s -----------------------------"
% (nItem, g.sCollectionName, g.sDatabaseName))
print(thing)
def fnnPutIterToDb(myitHandle,mysSeparator,myoCollection):
'''
Get lines of file from iterable, add to db.
First line of file is header with field names.
'''
nNewRecs = 0
lNames = []
for sLine in itertools.ifilter(fnbNotIgnoreLine, myitHandle):
# The first line is header containing field names.
# All other lines are data.
if lNames == []:
lNames = sLine.strip().split(mysSeparator)
NTRC.ntrace(3, "proc field names|{}|".format(lNames))
continue
lValues = map(fnIntPlease, sLine.strip().split(mysSeparator))
# Combine names and values into a list, then into a dict.
dLine = dict(zip(lNames,lValues))
NTRC.ntrace(3, "proc line dict|{}|".format(dLine))
# Store new record into db table (pardon me, collection).
oNewID = myoCollection.insert_one(dLine).inserted_id
nNewRecs += 1
NTRC.ntrace(3, "proc stored line n|{}| post_id|{}|".format(nNewRecs, oNewID))
NTRC.ntrace(3, "proc lines stored|{}|".format(myoCollection.count()))
return myoCollection.count()
def main(gl):
NTRC.ntrace(0, "Starting...")
tStart = datetime.datetime.now()
llFullOutput = mainNewBroker(gl)
if gl.bDebugPrint:
# Print all the crap that comes back.
print("---------begin cases----------")
for lCase in llFullOutput:
sCaseOut = ""
NTRC.ntrace(3, "proc fromq lCase|%s|" % (lCase))
sCaseOut = '\n'.join(lCase)
print(sCaseOut)
print("--------------")
print("---------end cases----------")
NTRC.ntrace(0, "Finished nWaitedForSlot|%s| nWaitedForDone|%s|"
% (gl.nWaitedForSlot, gl.nWaitedForDone))
tEnd = datetime.datetime.now()
tDif = tEnd - tStart
tDifMuSec = float((tDif.seconds * 1E6) + tDif.microseconds)
NTRC.ntrace(0, "Time total|%.3f|sec cases|%s| parallel|%s| "
"per case|%.0f|msec"
% (tDifMuSec/1E6, gl.nCases, gl.nParallel,
tDifMuSec/gl.nCases/1E3))
def main(gl):
NTRC.ntrace(0, "Starting...")
tStart = datetime.datetime.now()
llFullOutput = mainNewBroker(gl)
if gl.bDebugPrint:
# Print all the crap that comes back.
print("---------begin cases----------")
for lCase in llFullOutput:
sCaseOut = ""
NTRC.ntrace(3, "proc fromq lCase|%s|" % (lCase))
sCaseOut = '\n'.join(lCase)
print(sCaseOut)
print("--------------")
print("---------end cases----------")
NTRC.ntrace(0, "Finished nWaitedForSlot|%s| nWaitedForDone|%s|"
% (gl.nWaitedForSlot, gl.nWaitedForDone))
tEnd = datetime.datetime.now()
tDif = tEnd - tStart
tDifMuSec = float((tDif.seconds * 1E6) + tDif.microseconds)
NTRC.ntrace(0, "Time total|%.3f|sec cases|%s| parallel|%s| "
"per case|%.0f|msec"
% (tDifMuSec/1E6, gl.nCases, gl.nParallel,
tDifMuSec/gl.nCases/1E3))
def mainsim_post():
# C O L L E C T D A T A
# Collect all the bloody data, one item at a time, grumble.
sFamilyDir = request.forms.get("sFamilyDir")
sSpecificDir = request.forms.get("sSpecificDir")
sDatabaseName = request.forms.get("sDatabaseName")
nRandomSeeds = request.forms.get("nRandomSeeds")
lCopies= request.forms.getall("nCopies")
lLifem = request.forms.getall("nLifem")
nServerDefaultLife = request.forms.getall("nServerDefaultLife")
nAuditFreq = request.forms.get("nAuditFreq")
nAuditSegments = request.forms.get("nAuditSegments")
sAuditType = request.forms.get("sAuditType")
lGlitchFreq = request.forms.getall("nGlitchFreq")
lGlitchImpact = request.forms.getall("nGlitchImpact")
lGlitchMaxlife = request.forms.getall("nGlitchMaxlife")
nGlitchSpan = request.forms.get("nGlitchSpan")
nGlitchDecay = request.forms.get("nGlitchDecay")
lShockFreq = request.forms.getall("nShockFreq")
lShockImpact = request.forms.getall("nShockImpact")
lShockSpan = request.forms.getall("nShockSpan")
lShockMaxlife = request.forms.getall("nShockMaxlife")
nDocuments = request.forms.getall("nDocuments")
nDocSize = request.forms.get("nDocSize")
nShelfSize = request.forms.get("nShelfSize")
bShortLog = request.forms.get("bShortLog")
nSimLength = request.forms.getall("nSimLength")
nBandwidthMbps = request.forms.get("nBandwidthMbps")
bRedo = request.forms.get("bRedo")
bTestOnly = request.forms.get("bTestOnly")
bRunDetached = request.forms.get("bRunDetached")
sDetachedLogfile = request.forms.get("sDetachedLogfile")
msg = "mainsim_post: NOT YET IMPLEMENTED"
# F O R M D I C T I O N A R Y O F S U B S T I T U T I O N S
# Make a dictionary to use to substitute params into CLI command.
dVals = dict(sFamilyDir=sFamilyDir, sSpecificDir=sSpecificDir,
sCopies=fnsQuoteMulti(lCopies),
nServerDefaultLife=fnsQuoteMulti(nServerDefaultLife),
sLifem=fnsQuoteMulti(lLifem),
nAuditFreq=nAuditFreq, nAuditSegments=nAuditSegments,
sAuditType=sAuditType,
sGlitchFreq=fnsQuoteMulti(lGlitchFreq),
sGlitchImpact=fnsQuoteMulti(lGlitchImpact),
sGlitchMaxlife=fnsQuoteMulti(lGlitchMaxlife),
nGlitchSpan=nGlitchSpan,
nGlitchDecay=nGlitchDecay,
bShortLog=bShortLog,
nSimLength=fnsQuoteMulti(nSimLength),
nBandwidthMbps=nBandwidthMbps,
nRandomSeeds=nRandomSeeds,
sShockFreq=fnsQuoteMulti(lShockFreq),
sShockImpact=fnsQuoteMulti(lShockImpact),
sShockSpan=fnsQuoteMulti(lShockSpan),
sShockMaxlife=fnsQuoteMulti(lShockMaxlife),
nDocSize=nDocSize, nShelfSize=nShelfSize,
nDocuments=fnsQuoteMulti(nDocuments),
bRedo=bRedo,
bTestOnly=bTestOnly,
sDatabaseName=sDatabaseName,
bRunDetached=bRunDetached,
sDetachedLogfile=sDetachedLogfile,
msg=msg
)
NTRC.ntrace(3,"proc first dict|%s|" % (dVals))
# S P E C I A L C A S E S O F I N T E R D E P E N D E N C E
# If the user specified a logfile for detached running, then
# pretend that he remembered to check the box, too.
if dVals["sDetachedLogfile"]:
dVals["bRunDetached"] = True
# If the user wants to run detached, we may have to supply
# a default filename.
# Be sure to add today's date to the default filename.
if dVals["bRunDetached"] and not dVals["sDetachedLogfile"]:
dVals["sDetachedLogfile"] = ("./BrokerDetachedLogfile"
+ "_"
+ util.fnsGetTimeStamp().split("_")[0]
+ ".log")
# A D D E X T R A S P E C I F I C S T R I N G S
# If the user asks for a shortlog, add the option to the command.
dVals["xshortlog"] = "--shortlog" if bShortLog else ""
# If the user asks for a test list only, add that option to the command.
dVals["xtestonly"] = "--listonly" if bTestOnly else ""
# If the user asks for a rematch, add that option to the command.
dVals["xredo"] = "--redo" if bRedo else ""
# Format the Mongo range expression for nCopies
dVals["xcopies"] = dVals["sCopies"]
# Format the Mongo range expression for nLifem
dVals["xlifem"] = dVals["sLifem"]
NTRC.ntrace(3,"proc expanded dict|%s|" % (dVals))
# If running detached with output to a log file,
# specify append to file and detach process.
if dVals["sDetachedLogfile"]:
dVals["xLogfileExpr"] = (" >> "
+ dVals["sDetachedLogfile"]
+ " &")
else:
dVals["xLogfileExpr"] = ""
# S E L E C T C O M M A N D A N D F O R M A T I T
# Do something with the form data
sActualCli = cCmd.mGentlyFormat(sMainCommandStringToStdout, dVals)
# sActualCli = cCmd.mGentlyFormat(sMainCommandStringTestOnly, dVals)
# sActualCli = cCmd.mGentlyFormat(sMainCommandStringDumbTest, dVals)
NTRC.ntrace(3,"proc actual cli|%s|" % (sActualCli))
sPrefix = '''<html><body>
<font face="Courier">\n
'''
sPrefix += '<br />'
sPrefix += util.fnsGetTimeStamp()
sPrefix += ' Working. . . \n'
sPrefix += "<br />" + sActualCli + "<br />\n"
sSuffix = '''
</font>
</body></html>
'''
sLinePrefix = '<br />'
sLineSuffix = ''
# return dVals # DEBUG: return dict for visual inspection.
sResult = fnValidateDirectories(dVals)
if sResult:
response.status = 300
return sPrefix + sResult + sSuffix
# Other validations go in here.
if sResult == "":
# E X E C U T E C L I C M D , R E T U R N R E S U L T S
return cCmd.mDoCmdGen(sPrefix, sSuffix, sLinePrefix, sLineSuffix,
sActualCli)
def fniClearCollection(myoDb,mysCollectionName):
myoDb[mysCollectionName].remove() # Clear outmoldy, old records.
NTRC.ntrace(3, "Cleared collection {}.".format(mysCollectionName))
return myoDb[mysCollectionName].count()
def fnoOpenDb(mysDbName):
client = MongoClient('localhost', 27017)
db = client[mysDbName] # Create db if not already there.
NTRC.ntrace(3, "Connected to db.")
return db
def shockcall(timerobj, context):
NTRC.ntrace(0, "proc shock.call before setevent id|%s| t|%s|"
% (context, G.env.now))
timerobj.setevent()
'''
Interface of a few convenience functions for using pymongo.
- OpenDb (someone else has already started the mongod server)
- ClearCollection: remove data from a collection in db
- PutFileToDb: add CSV (or blank, tab, etc.) data to db
- PutIterToDb: add any iterable file to db
- NotIgnoreLine: used to filter blank and comment lines from input file
- IntPlease: convert string to int if possible
- GetSet: find in db collection (table) using some criteria in dict
- GetPendingWork: generate items from pending table that are not in done table
(more to come)
'''
from NewTraceFac import ntrace,ntracef,NTRC
NTRC.ntrace(3, "Importing...")
from pymongo import MongoClient
import re
import sys
import itertools
@ntrace
def fnoOpenDb(mysDbName):
client = MongoClient('localhost', 27017)
db = client[mysDbName] # Create db if not already there.
NTRC.ntrace(3, "Connected to db.")
return db
@ntrace
def fniClearCollection(myoDb,mysCollectionName):
myoDb[mysCollectionName].remove() # Clear outmoldy, old records.
def mReduceSingleServerLifetime(self, mysServerID, myfReduction):
'''
Reduce the lifetime of a single server.
Two possible methods, selected by a globaldata const nShockType.
- 1: lifetime, which was already a random from a distribution
with the standard server half-life, is then reduced
by some percentage during the shock period.
- 2: lifetime during the shock period is a new random
chosen from a distribution with half-life reduced
*from its current lifetime* by the shock percentage.
'''
cServer = G.dID2Server[mysServerID]
fCurrentLife = cServer.mfGetMyCurrentLife()
fOriginalLife = cServer.mfGetMyOriginalLife()
# Hack to experiment with the two types of shock to see if they
# are statistically different.
if G.nShockType == 1:
# Type 1: Lifetime during the shock period is the
# reduction of the original lifetime by the given
# percentage.
# That is, the server gets a single life expectation at
# birth, and it may be reduced by a shock and then
# restored at the end of the shock period, provided
# that it has not expired during the shock period.
fNewLifeParam = (1.0 - myfReduction) * fCurrentLife
# Lifetime cannot actually be zero for 100% reduction, so
# make it just really, really small, like 2 hours.
fNewLifeParam = max(fNewLifeParam, 2.0)
NTRC.ntracef(3, "SHOK", "proc shock1 at t|%8.0f| svr|%s| new"
"lifeparam|%.0f| shocktype|%s|"
% (G.env.now, mysServerID, fNewLifeParam, G.nShockType))
fNewLife = fNewLifeParam
elif G.nShockType == 2:
# Type 2: lifetime during shock period is a new
# random chosen from a distribution with less than the lifetime
# of the old one.
fNewLifeParam = (1.0 - myfReduction) * fOriginalLife
# Lifetime cannot actually be zero for 100% reduction, so
# make it just really, really small, like 2 hours.
fNewLifeParam = max(fNewLifeParam, 2.0)
NTRC.ntracef(3, "SHOK", "proc shock1 at t|%8.0f| svr|%s| new"
"lifeparam|%.0f| shocktype|%s|"
% (G.env.now, mysServerID, fNewLifeParam, G.nShockType))
fNewLife = util.makeserverlife(fNewLifeParam)
else:
NTRC.ntrace(0, "SHOK", "proc ERROR at t|%8.0f| svr|%s| "
"unknown shock type|%s|"
% (G.env.now, mysServerID, G.nShockType))
# Should throw a bugcheck fatal error at this point.
NTRC.ntracef(3, "SHOK", "proc shock2 at t|%8.0f| svr|%s| new"
"life|%.0f| shocktype|%s|"
% (G.env.now, mysServerID, fNewLife, G.nShockType))
lg.logInfo("SHOCK ", "t|%6.0f| reduce svr|%s| life by|%s| from|%.0f| to"
"|%.0f| shocktype|%s|"
% (G.env.now, mysServerID, myfReduction, fOriginalLife, fNewLife,
G.nShockType))
cServer.mRescheduleMyLife(fNewLife)
cServer.mSetServerInShock(True)
return
#!/usr/bin/python
'''
dblistdatabases.py
List all the databases Mongo holds on this system.
@author: rblandau
'''
from NewTraceFac import NTRC, ntrace, ntracef
import pymongo
NTRC.ntrace(0, "Begin.")
# Use naked Mongo functions not suitable for searchdatabasemongo library.
# Since MongoDB is a system-wide singleton resource, there is no need
# to get any name arguments for this command.
client = pymongo.MongoClient()
for sName in client.database_names():
print sName
NTRC.ntrace(0, "End.")
#END
def servercall(timerobj, context):
NTRC.ntrace(0, "proc server.call before setevent id|%s| t|%s|"
% (context.ID, G.env.now))
NTRC.ntrace(3,"proc server.call |%s| setting timer|%s| event|%s|"
% (context, timerobj, timerobj.event))
timerobj.setevent("OK")
def mReduceSingleServerLifetime(self, mysServerID, myfReduction):
'''
Reduce the lifetime of a single server.
Two possible methods, selected by a globaldata const nShockType.
- 1: lifetime, which was already a random from a distribution
with the standard server half-life, is then reduced
by some percentage during the shock period.
- 2: lifetime during the shock period is a new random
chosen from a distribution with half-life reduced
*from its current lifetime* by the shock percentage.
'''
cServer = G.dID2Server[mysServerID]
fCurrentLife = cServer.mfGetMyCurrentLife()
fOriginalLife = cServer.mfGetMyOriginalLife()
# Hack to experiment with the two types of shock to see if they
# are statistically different.
if G.nShockType == 1:
# Type 1: Lifetime during the shock period is the
# reduction of the original lifetime by the given
# percentage.
# That is, the server gets a single life expectation at
# birth, and it may be reduced by a shock and then
# restored at the end of the shock period, provided
# that it has not expired during the shock period.
fNewLifeParam = (1.0 - myfReduction) * fCurrentLife
# Lifetime cannot actually be zero for 100% reduction, so
# make it just really, really small, like 2 hours.
fNewLifeParam = max(fNewLifeParam, 2.0)
NTRC.ntracef(
3, "SHOK", "proc shock1 at t|%8.0f| svr|%s| new"
"lifeparam|%.0f| shocktype|%s|" %
(G.env.now, mysServerID, fNewLifeParam, G.nShockType))
fNewLife = fNewLifeParam
elif G.nShockType == 2:
# Type 2: lifetime during shock period is a new
# random chosen from a distribution with less than the lifetime
# of the old one.
fNewLifeParam = (1.0 - myfReduction) * fOriginalLife
# Lifetime cannot actually be zero for 100% reduction, so
# make it just really, really small, like 2 hours.
fNewLifeParam = max(fNewLifeParam, 2.0)
NTRC.ntracef(
3, "SHOK", "proc shock1 at t|%8.0f| svr|%s| new"
"lifeparam|%.0f| shocktype|%s|" %
(G.env.now, mysServerID, fNewLifeParam, G.nShockType))
fNewLife = util.makeserverlife(fNewLifeParam)
else:
NTRC.ntrace(
0, "SHOK", "proc ERROR at t|%8.0f| svr|%s| "
"unknown shock type|%s|" %
(G.env.now, mysServerID, G.nShockType))
# Should throw a bugcheck fatal error at this point.
NTRC.ntracef(
3, "SHOK", "proc shock2 at t|%8.0f| svr|%s| new"
"life|%.0f| shocktype|%s|" %
(G.env.now, mysServerID, fNewLife, G.nShockType))
lg.logInfo(
"SHOCK ", "t|%6.0f| reduce svr|%s| life by|%s| from|%.0f| to"
"|%.0f| shocktype|%s|" % (G.env.now, mysServerID, myfReduction,
fOriginalLife, fNewLife, G.nShockType))
cServer.mRescheduleMyLife(fNewLife)
cServer.mSetServerInShock(True)
return
def serverinter(timerobj, context):
NTRC.ntrace(0, "proc server.inter id|%s| t|%s|"
% (context.ID, G.env.now))
NTRC.ntrace(3,"proc server.inter |%s| canceling timer|%s| event|%s|"
% (context, timerobj, context.oldevent))
context.oldevent.succeed("CANCELED")
)
if 1:
TEMPLATE_FILE = sys.argv[1]
template = templateEnv.get_template( TEMPLATE_FILE )
sTitle = "Test page j8"
sSlugline = "<p>Trying to insert multiple vars with multiple values.</p>"
dPage= dict()
dPage["sTitle"] = sTitle
dPage["sSlugline"] = sSlugline
dIns = searchspace.fndReadAllInsFilesForGUI(
'./ins', '.ins3'
)
dVars = dIns
for (sVarname, dVardef) in dVars.items():
NTRC.ntrace(3,"proc onevar varname|%s| vardef|%s|" % (sVarname, dVardef))
dFinal = {"dVars" : dVars}
dFinal.update(dPage)
NTRC.ntrace(3,"proc dFinal|%s|" % (dFinal))
NTRC.ntrace(3,"proc dFinal.dVars|%s|" % (dFinal["dVars"]))
NTRC.ntrace(3,"proc dFinal.dVars.nCopies|%s|" % (dFinal['dVars']['nCopies']))
for varname,vardict in dFinal['dVars'].iteritems():
NTRC.ntrace(3,"proc dFinal.dVars.varname|%s| : |%s|" % (varname, vardict))
outputText = template.render( dFinal )
print(outputText)
def shockinter(timerobj, context):
NTRC.ntrace(0, "proc shock.inter id|%s| t|%s|"
% (context, G.env.now))
TEMPLATE_FILE = sInFile
template = templateEnv.get_template( TEMPLATE_FILE )
sTitle = "Test page d5"
sSlugline = "<p>Trying to insert multiple vars with multiple values.</p>"
dPage= dict()
dPage["sTitle"] = sTitle
dPage["sSlugline"] = sSlugline
dPage["sGeneratedTimestamp"] = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
# Process all instructions into dict.
dVars = searchspace.fndReadAllInsFilesForGUI(
sInsLoc, '.ins3'
)
for (sVarname, dVardef) in dVars.items():
NTRC.ntrace(3,"proc onevar varname|%s| vardef|%s|" % (sVarname, dVardef))
dFinal = {"dVars" : dVars}
dFinal.update(dPage)
NTRC.ntrace(3,"proc dFinal|%s|" % (dFinal))
# Process dict and forms thru Jinja2.
outputText = template.render( dFinal )
if sOutFile:
with open(sOutFile, 'w') as fhOut:
print >> fhOut, outputText
else:
print(outputText)
# Edit history
# 20170218 RBL Original version.
#!/usr/bin/python
'''
dblistdatabases.py
List all the databases Mongo holds on this system.
@author: rblandau
'''
from NewTraceFac import NTRC,ntrace,ntracef
import pymongo
NTRC.ntrace(0,"Begin.")
# Use naked Mongo functions not suitable for searchdatabasemongo library.
# Since MongoDB is a system-wide singleton resource, there is no need
# to get any name arguments for this command.
client = pymongo.MongoClient()
for sName in client.database_names():
print sName
NTRC.ntrace(0,"End.")
#END
)
if 1:
TEMPLATE_FILE = sys.argv[1]
template = templateEnv.get_template(TEMPLATE_FILE)
sTitle = "Test page j8"
sSlugline = "<p>Trying to insert multiple vars with multiple values.</p>"
dPage = dict()
dPage["sTitle"] = sTitle
dPage["sSlugline"] = sSlugline
dIns = searchspace.fndReadAllInsFilesForGUI('./ins', '.ins3')
dVars = dIns
for (sVarname, dVardef) in dVars.items():
NTRC.ntrace(3,
"proc onevar varname|%s| vardef|%s|" % (sVarname, dVardef))
dFinal = {"dVars": dVars}
dFinal.update(dPage)
NTRC.ntrace(3, "proc dFinal|%s|" % (dFinal))
NTRC.ntrace(3, "proc dFinal.dVars|%s|" % (dFinal["dVars"]))
NTRC.ntrace(3,
"proc dFinal.dVars.nCopies|%s|" % (dFinal['dVars']['nCopies']))
for varname, vardict in dFinal['dVars'].iteritems():
NTRC.ntrace(
3, "proc dFinal.dVars.varname|%s| : |%s|" % (varname, vardict))
outputText = template.render(dFinal)
print(outputText)
def mainsim_post():
# C O L L E C T D A T A
# Collect all the bloody data, one item at a time, grumble.
sFamilyDir = request.forms.get("sFamilyDir")
sSpecificDir = request.forms.get("sSpecificDir")
sDatabaseName = request.forms.get("sDatabaseName")
nRandomSeeds = request.forms.get("nRandomSeeds")
lCopies = request.forms.getall("nCopies")
lLifem = request.forms.getall("nLifem")
nServerDefaultLife = request.forms.getall("nServerDefaultLife")
nAuditFreq = request.forms.get("nAuditFreq")
nAuditSegments = request.forms.get("nAuditSegments")
sAuditType = request.forms.get("sAuditType")
lGlitchFreq = request.forms.getall("nGlitchFreq")
lGlitchImpact = request.forms.getall("nGlitchImpact")
lGlitchMaxlife = request.forms.getall("nGlitchMaxlife")
nGlitchSpan = request.forms.get("nGlitchSpan")
nGlitchDecay = request.forms.get("nGlitchDecay")
lShockFreq = request.forms.getall("nShockFreq")
lShockImpact = request.forms.getall("nShockImpact")
lShockSpan = request.forms.getall("nShockSpan")
lShockMaxlife = request.forms.getall("nShockMaxlife")
nDocuments = request.forms.getall("nDocuments")
nDocSize = request.forms.get("nDocSize")
nShelfSize = request.forms.get("nShelfSize")
bShortLog = request.forms.get("bShortLog")
nSimLength = request.forms.getall("nSimLength")
nBandwidthMbps = request.forms.get("nBandwidthMbps")
bRedo = request.forms.get("bRedo")
bTestOnly = request.forms.get("bTestOnly")
bRunDetached = request.forms.get("bRunDetached")
sDetachedLogfile = request.forms.get("sDetachedLogfile")
msg = "mainsim_post: NOT YET IMPLEMENTED"
# F O R M D I C T I O N A R Y O F S U B S T I T U T I O N S
# Make a dictionary to use to substitute params into CLI command.
dVals = dict(sFamilyDir=sFamilyDir,
sSpecificDir=sSpecificDir,
sCopies=fnsQuoteMulti(lCopies),
nServerDefaultLife=fnsQuoteMulti(nServerDefaultLife),
sLifem=fnsQuoteMulti(lLifem),
nAuditFreq=nAuditFreq,
nAuditSegments=nAuditSegments,
sAuditType=sAuditType,
sGlitchFreq=fnsQuoteMulti(lGlitchFreq),
sGlitchImpact=fnsQuoteMulti(lGlitchImpact),
sGlitchMaxlife=fnsQuoteMulti(lGlitchMaxlife),
nGlitchSpan=nGlitchSpan,
nGlitchDecay=nGlitchDecay,
bShortLog=bShortLog,
nSimLength=fnsQuoteMulti(nSimLength),
nBandwidthMbps=nBandwidthMbps,
nRandomSeeds=nRandomSeeds,
sShockFreq=fnsQuoteMulti(lShockFreq),
sShockImpact=fnsQuoteMulti(lShockImpact),
sShockSpan=fnsQuoteMulti(lShockSpan),
sShockMaxlife=fnsQuoteMulti(lShockMaxlife),
nDocSize=nDocSize,
nShelfSize=nShelfSize,
nDocuments=fnsQuoteMulti(nDocuments),
bRedo=bRedo,
bTestOnly=bTestOnly,
sDatabaseName=sDatabaseName,
bRunDetached=bRunDetached,
sDetachedLogfile=sDetachedLogfile,
msg=msg)
NTRC.ntrace(3, "proc first dict|%s|" % (dVals))
# S P E C I A L C A S E S O F I N T E R D E P E N D E N C E
# If the user specified a logfile for detached running, then
# pretend that he remembered to check the box, too.
if dVals["sDetachedLogfile"]:
dVals["bRunDetached"] = True
# If the user wants to run detached, we may have to supply
# a default filename.
# Be sure to add today's date to the default filename.
if dVals["bRunDetached"] and not dVals["sDetachedLogfile"]:
dVals["sDetachedLogfile"] = ("./BrokerDetachedLogfile" + "_" +
util.fnsGetTimeStamp().split("_")[0] +
".log")
# A D D E X T R A S P E C I F I C S T R I N G S
# If the user asks for a shortlog, add the option to the command.
dVals["xshortlog"] = "--shortlog" if bShortLog else ""
# If the user asks for a test list only, add that option to the command.
dVals["xtestonly"] = "--listonly" if bTestOnly else ""
# If the user asks for a rematch, add that option to the command.
dVals["xredo"] = "--redo" if bRedo else ""
# Format the Mongo range expression for nCopies
dVals["xcopies"] = dVals["sCopies"]
# Format the Mongo range expression for nLifem
dVals["xlifem"] = dVals["sLifem"]
NTRC.ntrace(3, "proc expanded dict|%s|" % (dVals))
# If running detached with output to a log file,
# specify append to file and detach process.
if dVals["sDetachedLogfile"]:
dVals["xLogfileExpr"] = (" >> " + dVals["sDetachedLogfile"] + " &")
else:
dVals["xLogfileExpr"] = ""
# S E L E C T C O M M A N D A N D F O R M A T I T
# Do something with the form data
sActualCli = cCmd.mGentlyFormat(sMainCommandStringToStdout, dVals)
# sActualCli = cCmd.mGentlyFormat(sMainCommandStringTestOnly, dVals)
# sActualCli = cCmd.mGentlyFormat(sMainCommandStringDumbTest, dVals)
NTRC.ntrace(3, "proc actual cli|%s|" % (sActualCli))
sPrefix = '''<html><body>
<font face="Courier">\n
'''
sPrefix += '<br />'
sPrefix += util.fnsGetTimeStamp()
sPrefix += ' Working. . . \n'
sPrefix += "<br />" + sActualCli + "<br />\n"
sSuffix = '''
</font>
</body></html>
'''
sLinePrefix = '<br />'
sLineSuffix = ''
# return dVals # DEBUG: return dict for visual inspection.
sResult = fnValidateDirectories(dVals)
if sResult:
response.status = 300
return sPrefix + sResult + sSuffix
# Other validations go in here.
if sResult == "":
# E X E C U T E C L I C M D , R E T U R N R E S U L T S
return cCmd.mDoCmdGen(sPrefix, sSuffix, sLinePrefix, sLineSuffix,
sActualCli)