def readParams( ParamsDic ):
""" read params, return a SWFParams class, with new fields """
CurrentSWFParams = SWFParams()
CurrentSWFParams.SWF = SWFOnlyParams()
if 'otherinfo' in ParamsDic.keys():
SWFConstsInstance = SWFConsts()
try:
for key in ParamsDic['otherinfo']:
OneList = key.split('.')
if OneList[0] == 'SWF':
if OneList[1] == 'File':
CurrentSWFParams.SWF.File = ParamsDic['otherinfo'][key]
elif OneList[1] == 'Scale': # SWF.Scale.SubmitTime=1:1
KeyID = SWFConstsInstance.getID(OneList[2])
if (KeyID is not None) and (KeyID >= 0):
# save: ['SubmitTime']: (1,1)
ScaleIn, ScaleOut = ParamsDic['otherinfo'][key].split(':')
ScaleIn = float(ScaleIn)
ScaleOut = float(ScaleOut)
CurrentSWFParams.SWF.Scale[OneList[2]] = (ScaleIn, ScaleOut)
elif OneList[1] == 'Filter':
KeyID = SWFConstsInstance.getID(OneList[2])
##print 'Filter', OneList[2], "KeyID=", KeyID
if (KeyID is not None) and (KeyID >= 0):
#SWF.Filter.JobNumber.Min=31
# save: ['JobNumber']: ('Min',31)
if OneList[2] not in CurrentSWFParams.SWF.Filters:
CurrentSWFParams.SWF.Filters[OneList[2]] = []
CurrentSWFParams.SWF.Filters[OneList[2]].append((OneList[3], int(ParamsDic['otherinfo'][key])))
else:
# special cases
##print "Special case filter", OneList[2]
# SWF.Filter.Delta.SubmitTime.Hours.Max=3
# save: ['Delta']['SubmitTime']['Hours']('Max',31)
if OneList[2] == 'Delta':
if 'Delta' not in CurrentSWFParams.SWF.Filters:
CurrentSWFParams.SWF.Filters['Delta'] = {}
KeyID = SWFConstsInstance.getID(OneList[3])
if KeyID and KeyID >= 0:
if OneList[3] not in CurrentSWFParams.SWF.Filters['Delta']:
CurrentSWFParams.SWF.Filters['Delta'][OneList[3]] = {}
CurrentSWFParams.SWF.Filters['Delta'][OneList[3]][OneList[4]] = (OneList[5], int(ParamsDic['otherinfo'][key]))
# SWF.Filter.NJobs.Max=1000
# save: ['NJobs']: ('Max',1000)
if OneList[2] == 'NJobs':
CurrentSWFParams.SWF.Filters['NJobs'] = (OneList[3], int(ParamsDic['otherinfo'][key]))
except Exception, e:
print "Exception", e
print traceback.print_exc()
raise Exception, "SWF generator: could not decode " + str(key) + '.'
if CurrentSWFParams.SWF.File is None:
raise Exception, "SWF generator: No SWF.File field in OtherInfo (cannot read an unknown file)."
# UseArrivalTimeDistribution=false
try:
CurrentSWFParams.UseArrivalTimeDistribution = AIParseUtils.readBoolean(ParamsDic['otherinfo']['UseArrivalTimeDistribution'])
except KeyError:
CurrentSWFParams.UseArrivalTimeDistribution = 0
#Application=swf
#AppBaseDir=/tmp
try:
CurrentSWFParams.AppBaseDir = ParamsDic['otherinfo']['AppBaseDir']
except KeyError:
CurrentSWFParams.AppBaseDir = '/tmp'
#raise Exception, "SMPI1 generator: No AppBaseDir info specified in the OtherInfo field."
print "WARNING! SWF generator: No AppBaseDir info specified in the OtherInfo field.", "Assuming default:", AppBaseDir
try:
CurrentSWFParams.Application = ParamsDic['otherinfo']['Application']
except KeyError:
CurrentSWFParams.Application = 'swf'
#raise Exception, "SMPI1 generator: No AppBaseDir info specified in the OtherInfo field."
print "WARNING! SWF generator: No Application info specified in the OtherInfo field.", "Assuming default:", Application
try:
CurrentSWFParams.Submitter = ParamsDic['otherinfo']['Submitter']
except KeyError:
CurrentSWFParams.Submitter = 'krunner -l DEBUG -g -e -o -f ${JDF}'
print "WARNING! SWF generator: No Submitter info specified in the OtherInfo field.", "Assuming default", Submitter
try:
CurrentSWFParams.NComponentsWithWeightsDic = \
AIParseUtils.readIntWithWeightsList(
ParamsDic['otherinfo']['NComponentsWithWeights'],
DefaultWeight = 1.0
)
# NComponentsWithWeightsDic['Values']; NComponentsWithWeightsDic['TotalWeight']
except KeyError:
raise Exception, "SWF generator: Invalid NComponentsWithWeights info specified in the OtherInfo field."
##print ">>>>>> NComponentsWithWeightsDic", NComponentsWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
try:
CurrentSWFParams.MinComponentSize = AIParseUtils.readInt(ParamsDic['otherinfo']['MinComponentSize'], DefaultValue = 0)
except KeyError:
CurrentSWFParams.MinComponentSize = 1
print "WARNING! SWF generator: No MinComponentSize info specified in the OtherInfo field.", "Assuming default", CurrentSWFParams.MinComponentSize
#raise Exception, "SWF generator: No MinComponentSize info specified in the OtherInfo field."
try:
CurrentSWFParams.DelayBetweenUnits = AIParseUtils.readInt(ParamsDic['otherinfo']['DelayBetweenUnits'], DefaultValue = 0)
except KeyError:
CurrentSWFParams.DelayBetweenUnits = 1
print "WARNING! SWF generator: No DelayBetweenUnits info specified in the OtherInfo field.", "Assuming default", CurrentSWFParams.DelayBetweenUnits
try:
CurrentSWFParams.MaxWallTime = AIParseUtils.readInt(ParamsDic['otherinfo']['MaxWallTime'], DefaultValue = 1000000)
except KeyError:
CurrentSWFParams.MaxWallTime = 1
print "WARNING! SWF generator: No DelayBetweenUnits info specified in the OtherInfo field.", "Assuming default", CurrentSWFParams.MaxWallTime
try:
CurrentSWFParams.MaxComponentSize = AIParseUtils.readInt(ParamsDic['otherinfo']['MaxComponentSize'], DefaultValue = 0)
except KeyError:
CurrentSWFParams.MaxComponentSize = 10000000
print "WARNING! SWF generator: No MaxComponentSize info specified in the OtherInfo field.", "Assuming default", CurrentSWFParams.MaxComponentSize
#raise Exception, "SWF generator: No MaxComponentSize info specified in the OtherInfo field."
try:
CurrentSWFParams.EqualCPUsPerComponent = AIParseUtils.readBoolean(ParamsDic['otherinfo']['SWFAppsDir'])
except KeyError:
CurrentSWFParams.EqualCPUsPerComponent = 0
#raise Exception, "SWF generator: No EqualCPUsPerComponent info specified in the OtherInfo field. Setting to default " + AIParseUtils.IntToBooleanDic[EqualCPUsPerComponent] + '.'
try:
CurrentSWFParams.SiteTypesWithWeightsDic = \
AIParseUtils.readStringWithWeightsList(
ParamsDic['otherinfo']['SiteTypesWithWeights'],
DefaultWeight = 1.0
)
except KeyError:
CurrentSWFParams.SiteTypesWithWeights = {AIParseUtils.VALUES:[], AIParseUtils.TOTAL_WEIGHT:0.0 }
print "WARNING! SWF generator: No SiteTypesWithWeights info specified in the OtherInfo field.\n\tSetting to default..."
##print ">>>>>> SiteTypesWithWeightsDic", SiteTypesWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
try:
CurrentSWFParams.SitesWithWeightsDic = \
AIParseUtils.readStringWithWeightsList(
ParamsDic['otherinfo']['SitesWithWeights'],
DefaultWeight = 1.0
)
except KeyError:
CurrentSWFParams.SitesWithWeightsDic = {AIParseUtils.VALUES:[], AIParseUtils.TOTAL_WEIGHT:0.0 }
print "WARNING! SWF generator: No SitesWithWeights info specified in the OtherInfo field.\n\tSetting to default..."
def generateWorkloadUnit( UnitDir, UnitID, WLUnit, SubmitDurationMS, bGenerateRandom = 1 ):
"""
Out:
UnitsDic
o A dictionary with keys 'info' and 'jobs'.
Notes:
o UnitsDic['info'] contains a dictionary of jobs info,
indexed with an integer counter
o Each job info contains at least the keys
name, description, jdf, submitCommand, runTime
o UnitsDic['jobs'] contains a dictionary of jobs data,
indexed with an integer counter
o Each job data contains the list of components of that job
o Each component in the list is a dictionary with at least the
following fields: executable, stdout, stderr, name, description,
directory, maxWallTime, arguments, env, stagein, stageout
See also:
utils/WLDocHandlers.WLSubmitJobKeys
this_file.SMPI1Component.generateComponent
"""
global SMPI1_LastRunTime
SMPI1Component.SMPI1_Exe = os.path.join( "/tmp", "smpi1t" )
SMPI1Component.SMPI1_ParKSizes = [ "0", "32", "128", "512", "1024" ]
SMPI1Component.SMPI1_ParKSuperSizes = [ "0", "16", "32", "64", "128", "256" ]
SMPI1Component.SMPI1_ParSupersteps = [ "1", "2", "5", "10", "20", "50", "100" ]
SMPI1Component.SMPI1_ParMemoryKItems = [ "10", "25", "50", "100", "250", "500", "1000", "5000" ]
SMPI1Component.SMPI1_ParMemoryElementsPerItem = [ "3", "4", "10", "100", "500", "1000" ]
SMPI1Component.SMPI1_ParXChangeElementsPerStep = [ "100", "500", "1000", "10000", "50000" ]
SMPI1Component.SMPI1_ParComputationPerMemoryItem = [ "2", "10", "100", "1000" ]
SMPI1Component.SMPI1_RunTimeInMinutes = [ "2", "5", "10", "15" ]
if 'otherinfo' in WLUnit.keys():
try:
OneString = WLUnit['otherinfo']['Exe']
SMPI1Component.SMPI1_Exe = OneString
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParKSizes'],
ItemSeparator = ',' )
SMPI1Component.SMPI1_ParKSizes = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParKSuperSizes'],
ItemSeparator = ',' )
SMPI1Component.SMPI1_ParKSuperSizes = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParSupersteps'],
ItemSeparator = ',' )
SMPI1Component.SMPI1_ParSupersteps = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParMemoryKItems'],
ItemSeparator = ',' )
SMPI1Component.SMPI1_ParMemoryKItems = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParMemoryElementsPerItem'],
ItemSeparator = ',' )
SMPI1Component.SMPI1_ParMemoryElementsPerItem = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParXChangeElementsPerStep'],
ItemSeparator = ',' )
SMPI1Component.SMPI1_ParXChangeElementsPerStep = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParComputationPerMemoryItem'],
ItemSeparator = ',' )
SMPI1Component.SMPI1_ParComputationPerMemoryItem = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['RunTimeInMinutes'],
ItemSeparator = ',' )
SMPI1Component.SMPI1_RunTimeInMinutes = OneIntList
except KeyError:
pass
try:
NComponentsWithWeightsDic = \
AIParseUtils.readIntWithWeightsList(
WLUnit['otherinfo']['NComponentsWithWeights'],
DefaultWeight = 1.0
)
# NComponentsWithWeightsDic['Values']; NComponentsWithWeightsDic['TotalWeight']
except KeyError:
raise Exception, "SMPI1 generator: Invalid NComponentsWithWeights info specified in the OtherInfo field."
##print ">>>>>> NComponentsWithWeightsDic", NComponentsWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
try:
TotalCPUsWithWeightsDic = \
AIParseUtils.readIntWithWeightsList(
WLUnit['otherinfo']['TotalCPUsWithWeights'],
DefaultWeight = 1.0
)
except KeyError:
raise Exception, "SMPI1 generator: Invalid TotalCPUsWithWeights info specified in the OtherInfo field."
##print ">>>>>> TotalCPUsWithWeightsDic", TotalCPUsWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
try:
MinComponentSize = AIParseUtils.readInt(WLUnit['otherinfo']['MinComponentSize'], DefaultValue = 0)
except KeyError:
MinComponentSize = 1
#raise Exception, "SMPI1 generator: No MinComponentSize info specified in the OtherInfo field."
try:
MaxComponentSize = AIParseUtils.readInt(WLUnit['otherinfo']['MaxComponentSize'], DefaultValue = 0)
except KeyError:
MaxComponentSize = 10000000
#raise Exception, "SMPI1 generator: No MaxComponentSize info specified in the OtherInfo field."
try:
EqualCPUsPerComponent = AIParseUtils.readBoolean(WLUnit['otherinfo']['SMPI1AppsDir'])
except KeyError:
EqualCPUsPerComponent = 0
#raise Exception, "SMPI1 generator: No EqualCPUsPerComponent info specified in the OtherInfo field. Setting to default " + AIParseUtils.IntToBooleanDic[EqualCPUsPerComponent] + '.'
try:
AppBaseDir = WLUnit['otherinfo']['AppBaseDir']
except KeyError:
AppBaseDir = '/tmp'
#raise Exception, "SMPI1 generator: No AppBaseDir info specified in the OtherInfo field."
print "WARNING! SMPI1 generator: No AppBaseDir info specified in the OtherInfo field.", "Assuming default:", AppBaseDir
try:
Application = WLUnit['otherinfo']['Application']
except KeyError:
Application = 'smpi1t-gm'
#raise Exception, "SMPI1 generator: No AppBaseDir info specified in the OtherInfo field."
print "WARNING! SMPI1 generator: No Application info specified in the OtherInfo field.", "Assuming default:", Application
try:
Submitter = WLUnit['otherinfo']['Submitter']
except KeyError:
Submitter = 'krunner -l DEBUG -g -e -o -f ${JDF}'
print "WARNING! SMPI1 generator: No Submitter info specified in the OtherInfo field.", "Assuming default", Submitter
try:
SiteTypesWithWeightsDic = \
AIParseUtils.readStringWithWeightsList(
WLUnit['otherinfo']['SiteTypesWithWeights'],
DefaultWeight = 1.0
)
except KeyError:
SiteTypesWithWeights = {AIParseUtils.VALUES:[], AIParseUtils.TOTAL_WEIGHT:0.0 }
print "WARNING! SMPI1 generator: No SiteTypesWithWeights info specified in the OtherInfo field.\n\tSetting to default", SitesList
##print ">>>>>> SiteTypesWithWeightsDic", SiteTypesWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
try:
SitesWithWeightsDic = \
AIParseUtils.readStringWithWeightsList(
WLUnit['otherinfo']['SitesWithWeights'],
DefaultWeight = 1.0
)
except KeyError:
SitesWithWeightsDic = {AIParseUtils.VALUES:[], AIParseUtils.TOTAL_WEIGHT:0.0 }
print "WARNING! SMPI1 generator: No SitesWithWeights info specified in the OtherInfo field.\n\tSetting to default", SitesList
##print ">>>>>> SitesWithWeightsDic", SitesWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
else:
raise Exception, "SMPI1 generator: No OtherInfo data! Expected at least ComponentSize,NJobs,NComponents fields."
UnitsDic = {}
UnitsDic['info'] = {}
UnitsDic['jobs'] = {}
#-- init start time
try:
StartAt = AIParseUtils.readInt(WLUnit['otherinfo']['StartAt'], DefaultValue = 0)
StartAt = StartAt * 1000 # the time is given is seconds
except KeyError:
StartAt = 0
SMPI1_LastRunTime = StartAt
if 'FirstJobIndex' in WLUnit:
JobIndex = int(WLUnit['FirstJobIndex'])
else:
JobIndex = 0
WLUnitMultiplicity = int(WLUnit['multiplicity'])
if bGenerateRandom > 0 :
index = 0
while index < WLUnitMultiplicity:
# 1. generate a random site type (unordered or ordered)
SiteType = AIRandomUtils.getRandomWeightedListElement(
SiteTypesWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
# 2. select the number of CPU for this job
#TODO: repeat NCPUs selection until App can run on that many CPUs
NCPUs = AIRandomUtils.getRandomWeightedListElement(
TotalCPUsWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
# 3. select the number of components and try to create subcomponents,
# until successfully matching restrictions
WhileCondition = 1
NTries = 100
while WhileCondition != 0 and NTries > 0:
NTries = NTries - 1
NComponents = AIRandomUtils.getRandomWeightedListElement(
NComponentsWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
if NComponents > NCPUs:
continue # impossible match, just retry
NCPUsPerComponent = NCPUs / NComponents
# ensure restrictions are met
if NCPUsPerComponent < MinComponentSize or NCPUsPerComponent > MaxComponentSize:
continue
NRemainingComponents = NCPUs - NCPUsPerComponent * NComponents
if NRemainingComponents > 0 and EqualCPUsPerComponent == 1:
continue
# >= -> including the extra CPU for some components
if NRemainingComponents > 0 and NCPUsPerComponent == MaxComponentSize:
continue
PreComponentsList = []
for ComponentID in xrange(NComponents):
# cpu count
ToAssignHere = NCPUsPerComponent
if ComponentID < NRemainingComponents: # assign the extra CPUs, round-robin
ToAssignHere = ToAssignHere + 1
# site location
if SiteType == 'ordered':
Site = AIRandomUtils.getRandomWeightedListElement(
SitesWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
else:
Site = '*'
# create pre-component information
PreComponent = {}
PreComponent['id'] = ComponentID
PreComponent['count'] = ToAssignHere
PreComponent['location'] = Site
# add pre-component
PreComponentsList.append(PreComponent)
WhileCondition = 0
if WhileCondition == 1: # failed to generate in 100 tries
continue # try again for the same job
#--- generate rnd job
#-- generate job info
UnitsDic['info'][index] = \
generateJobInfo( UnitDir, UnitID, JobIndex, \
WLUnit, SubmitDurationMS, Submitter )
#-- generate job components
UnitsDic['jobs'][index] = \
generateJobComponents( UnitDir, UnitID, JobIndex, WLUnit, PreComponentsList, \
Application, AppBaseDir, \
bGenerateRandom )
index = index + 1
JobIndex = JobIndex + 1
WLUnit['generatedjobs'] = index
else:
index = 0
for Size in SMPI1Component.SMPI1_ParKSuperSizes:
for N in SMPI1Component.SMPI1_ParSupersteps:
for M in SMPI1Component.SMPI1_ParMemoryKItems:
for S in SMPI1Component.SMPI1_ParMemoryElementsPerItem:
for X in SMPI1Component.SMPI1_ParXChangeElementsPerStep:
for C in SMPI1Component.SMPI1_ParComputationPerMemoryItem:
raise Exception, "ERROR! SMPI1 is not finished!"
#--- generate wall time once for all components of this job
MaxWallTime = AIRandomUtils.getRandomListElement( SMPI1Component.SMPI1_RunTimeInSeconds )
#-- generate a random site type (unordered, ordered, ...)
SiteType = AIRandomUtils.getRandomWeightedListElement(
SiteTypesWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
#-- generate job info
UnitsDic['info'][index] = \
generateJobInfo( UnitDir, UnitID, JobIndex, \
WLUnit, SubmitDurationMS )
#-- generate job components
UnitsDic['jobs'][index] = \
generateJobComponents( UnitDir, UnitID, JobIndex, WLUnit, \
bGenerateRandom, SiteType, Size, \
N, M, S, X, C, MaxWallTime )
index = index + 1
JobIndex = JobIndex + 1
#-- set generated jobs
WLUnit['generatedjobs'] = index
return UnitsDic
for name in JDFGeneratorNames:
ResManagerName = name.replace('-jdf', '')
DirName = "jdfs-%s" % ResManagerName
OutDirName = WLOutDir.replace('jdfs', DirName)
if not os.path.exists(OutDirName):
os.makedirs(OutDirName)
except OSError, e:
print "Cannot create output directory for unit", WLUnitID, "...skipping unit"
print '\tOS returned:', e
continue
if 'RandomWorkload' in WLUnit['otherinfo'].keys():
try:
##print "Getting random info from", WLUnit['otherinfo']['RandomWorkload']
bGenerateRandom = AIParseUtils.readBoolean(WLUnit['otherinfo']['RandomWorkload'], 1)
except Exception, e:
print "Wrong value for RandomWorkload", "("+WLUnit['otherinfo']['RandomWorkload'].lower()+"). Expected true/false. Setting to default (true)"
bGenerateRandom = 1
else:
print "No value for RandomWorkload", "Setting to default (true)."
bGenerateRandom = 1
WLUnit['arrivaltimefunc'] = ArrivalTimeRandom.randomVariable
#--- get appropriate generator function
#--- v1.1: composite
# composite?
if WLUnit['composite'] == 1:
bGenerateRandom = 1 ### composite apps are always randomly generated
def generateWorkload( UnitDir, UnitID, WLUnit, SubmitDurationMS, bGenerateRandom = 1 ):
global IBIS_LastRunTime
#print WLUnit['otherinfo']
#{'RandomWorkload': 'true', 'NComponents': '1-8', 'NJobs': '50', 'ComponentSize':'4-8'}
if bGenerateRandom > 0:
print "Random workload"
else:
print "Full workload"
if 'otherinfo' in WLUnit.keys():
try:
##print ">>>>>> NJobs"
NJobs = AIParseUtils.readInt(WLUnit['otherinfo']['NJobs'], DefaultValue = 0)
except KeyError:
NJobs = 0
if bGenerateRandom > 0: #-- NJobs required if random, optional otherwise
raise Exception, "IBIS generator: No NJobs info specified in the OtherInfo field."
try:
NComponentsWithWeightsDic = \
AIParseUtils.readIntWithWeightsList(
WLUnit['otherinfo']['NComponentsWithWeights'],
DefaultWeight = 1.0
)
# NComponentsWithWeightsDic['Values']; NComponentsWithWeightsDic['TotalWeight']
except KeyError:
raise Exception, "IBIS generator: Invalid NComponentsWithWeights info specified in the OtherInfo field."
##print ">>>>>> NComponentsWithWeightsDic", NComponentsWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
try:
TotalCPUsWithWeightsDic = \
AIParseUtils.readIntWithWeightsList(
WLUnit['otherinfo']['TotalCPUsWithWeights'],
DefaultWeight = 1.0
)
except KeyError:
raise Exception, "IBIS generator: Invalid TotalCPUsWithWeights info specified in the OtherInfo field."
##print ">>>>>> TotalCPUsWithWeightsDic", TotalCPUsWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
try:
MinComponentSize = AIParseUtils.readInt(WLUnit['otherinfo']['MinComponentSize'], DefaultValue = 0)
except KeyError:
MinComponentSize = 1
#raise Exception, "IBIS generator: No MinComponentSize info specified in the OtherInfo field."
try:
MaxComponentSize = AIParseUtils.readInt(WLUnit['otherinfo']['MaxComponentSize'], DefaultValue = 0)
except KeyError:
MaxComponentSize = 10000000
#raise Exception, "IBIS generator: No MaxComponentSize info specified in the OtherInfo field."
try:
EqualCPUsPerComponent = AIParseUtils.readBoolean(WLUnit['otherinfo']['IbisAppsDir'])
except KeyError:
EqualCPUsPerComponent = 0
#raise Exception, "IBIS generator: No EqualCPUsPerComponent info specified in the OtherInfo field. Setting to default " + AIParseUtils.IntToBooleanDic[EqualCPUsPerComponent] + '.'
try:
IbisAppsDir = WLUnit['otherinfo']['IbisAppsDir']
except KeyError:
raise Exception, "IBIS generator: No IbisAppsDir info specified in the OtherInfo field."
try:
IbisLibDir = WLUnit['otherinfo']['IbisLibDir']
except KeyError:
raise Exception, "IBIS generator: No IbisLibDir info specified in the OtherInfo field."
try:
JavaHomeDir = WLUnit['otherinfo']['JavaHomeDir']
except KeyError:
raise Exception, "IBIS generator: No JavaHomeDir info specified in the OtherInfo field."
try:
AppsWithWeightsDic = \
AIParseUtils.readStringWithWeightsList(
WLUnit['otherinfo']['AppsWithWeights'],
DefaultWeight = 1.0
)
except KeyError:
raise Exception, "IBIS generator: Invalid AppsWithWeights info specified in the OtherInfo field."
###print ">>>>>> AppsWithWeightsDic", AppsWithWeightsDic#[AIParseUtils.TOTAL_WEIGHT]
try:
Submitter = WLUnit['otherinfo']['Submitter']
except KeyError:
Submitter = 'grunner'
print "WARNING! IBIS generator: No Submitter info specified in the OtherInfo field.\n\tSetting to default", Submitter
try:
SiteTypesWithWeightsDic = \
AIParseUtils.readStringWithWeightsList(
WLUnit['otherinfo']['SiteTypesWithWeights'],
DefaultWeight = 1.0
)
except KeyError:
SiteTypesWithWeights = {AIParseUtils.VALUES:[], AIParseUtils.TOTAL_WEIGHT:0.0 }
print "WARNING! IBIS generator: No SiteTypesWithWeights info specified in the OtherInfo field.\n\tSetting to default", SitesList
##print ">>>>>> SiteTypesWithWeightsDic", SiteTypesWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
try:
SitesWithWeightsDic = \
AIParseUtils.readStringWithWeightsList(
WLUnit['otherinfo']['SitesWithWeights'],
DefaultWeight = 1.0
)
except KeyError:
SitesWithWeightsDic = {AIParseUtils.VALUES:[], AIParseUtils.TOTAL_WEIGHT:0.0 }
print "WARNING! IBIS generator: No SitesWithWeights info specified in the OtherInfo field.\n\tSetting to default", SitesList
##print ">>>>>> SitesWithWeightsDic", SitesWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
try:
JavaMaxMemoryLimitMB = AIParseUtils.readInt(WLUnit['otherinfo']['JavaMaxMemoryLimitMB'], 10000)
except KeyError:
JavaMaxMemoryLimitMB = 10000
print "WARNING! IBIS generator: No JavaMaxMemoryLimitMB info specified in the OtherInfo field.\n\tSetting to default", JavaMaxMemoryLimitMB, "MB"
try:
SelectionMethodName = WLUnit['otherinfo']['SelectionMethod']
except KeyError:
SelectionMethodName = 'R'
raise Exception, "IBIS generator: No SelectionMethod info specified in the OtherInfo field."
else:
raise Exception, "IBIS generator: No OtherInfo data! Expected at least ComponentSize,NJobs,NComponents fields."
UnitsDic = {}
UnitsDic['info'] = {}
UnitsDic['jobs'] = {}
#-- init start time
try:
StartAt = AIParseUtils.readInt(WLUnit['otherinfo']['StartAt'], DefaultValue = 0)
StartAt = StartAt * 1000 # the time is given is seconds
except KeyError:
StartAt = 0
IBIS_LastRunTime = StartAt
WLUnitMultiplicity = int(WLUnit['multiplicity'])
if bGenerateRandom > 0:
#--- generate random
OneIBISSelectionMethod = IBISSelectionMethod( AppsWithWeightsDic )
OneIBISSelectionMethod.setSelectionMethodByName( SelectionMethodName )
JobIndex = 0
if WLUnitMultiplicity > 1:
NJobs = NJobs * WLUnitMultiplicity
while JobIndex < NJobs:
# 1. select application type
## Application = AIRandomUtils.getRandomWeightedListElement(
## AppsWithWeightsDic,
## ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
## )
Application = OneIBISSelectionMethod.getNextValue()
if Application.find('*') >= 0:
print "IBIS generator: Expanded", Application, "to",
Application = ExpandApplication(Application)
print Application
# 2. select the number of CPU for this job
#TODO: repeat NCPUs selection until App can run on that many CPUs
NCPUs = AIRandomUtils.getRandomWeightedListElement(
TotalCPUsWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
# 3. select the number of components and try to create subcomponents,
# until successfully matching restrictions
WhileCondition = 1
NTries = 100
while WhileCondition != 0 and NTries > 0:
NTries = NTries - 1
NComponents = AIRandomUtils.getRandomWeightedListElement(
NComponentsWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
if NComponents > NCPUs:
continue # impossible match, just retry
NCPUsPerComponent = NCPUs / NComponents
# ensure restrictions are met
if NCPUsPerComponent < MinComponentSize or NCPUsPerComponent > MaxComponentSize:
continue
NRemainingComponents = NCPUs - NCPUsPerComponent * NComponents
if NRemainingComponents > 0 and EqualCPUsPerComponent == 1:
continue
# >= -> including the extra CPU for some components
if NRemainingComponents > 0 and NCPUsPerComponent == MaxComponentSize:
continue
SiteType = AIRandomUtils.getRandomWeightedListElement(
SiteTypesWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
PreComponentsList = []
for ComponentID in xrange(NComponents):
# cpu count
ToAssignHere = NCPUsPerComponent
if ComponentID < NRemainingComponents: # assign the extra CPUs, round-robin
ToAssignHere = ToAssignHere + 1
# site location
if SiteType == 'ordered':
Site = AIRandomUtils.getRandomWeightedListElement(
SitesWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
else:
Site = '*'
# create pre-component information
PreComponent = {}
PreComponent['id'] = ComponentID
PreComponent['count'] = ToAssignHere
PreComponent['location'] = Site
# add pre-component
PreComponentsList.append(PreComponent)
WhileCondition = 0
if WhileCondition == 1: # failed to generate in 100 tries
continue # try again for the same job
#--- generate rnd job
#-- generate rnd job: ID
CurrentUnitID = "%s-%d" % (UnitID, JobIndex % WLUnitMultiplicity)
#-- generate rnd job: index
UnitsDic['info'][JobIndex] = \
generateJobInfo( UnitDir, CurrentUnitID, JobIndex, Application, \
WLUnit, SubmitDurationMS, Submitter )
#-- generate rnd job: components
UnitsDic['jobs'][JobIndex] = \
generateJobComponents(
UnitDir, CurrentUnitID, JobIndex, WLUnit,
PreComponentsList, JavaHomeDir, IbisAppsDir, IbisLibDir,
JavaMaxMemoryLimitMB, Application, NCPUs
)
#-- be more descriptive
UnitsDic['info'][JobIndex]['description'] = \
UnitsDic['info'][JobIndex]['description'] + \
" Components: %d. NCPUs: %d." % (len(PreComponentsList), NCPUs)
#--- done generating
if UnitsDic['jobs'][JobIndex] == None:
NJobs = NJobs - 1 #-- don't ack job, but make sure total no of jobs is decreased
else:
print "IBIS generator: Generated a job of type", Application,
print "with", len(PreComponentsList), "components/",
print NCPUs, "CPUs, in", UnitsDic['info'][JobIndex]['jdf']
JobIndex = JobIndex + 1 #-- ack job
WLUnit['generatedjobs'] = JobIndex
else:
print "Complete workload"
#--- generate all
JobIndex = 0
RepetitionIndex = 0
while RepetitionIndex < WLUnitMultiplicity:
for ComponentSize in ComponentSizeList:
for NumComponents in NumComponentsList:
CurrentUnitID = "%s-%d" % (UnitID, JobIndex % WLUnitMultiplicity)
UnitsDic['info'][JobIndex] = \
generateJobInfo( UnitDir, CurrentUnitID, JobIndex, \
WLUnit, SubmitDurationMS, Submitter )
UnitsDic['jobs'][JobIndex] = \
generateJobComponents( UnitDir, CurrentUnitID, JobIndex, WLUnit, \
NumComponents, ComponentSize, Application, NCPUs )
JobIndex = JobIndex + 1
RepetitionIndex = RepetitionIndex + 1
if JobIndex > NJobs and NJobs > 0:
print "Generated %d jobs and stopped because of the NJobs limit."
break
WLUnit['generatedjobs'] = JobIndex
#raise Exception, "IBIS generator: Not implemented yet"
return UnitsDic
