def readAllParams( WLUnit ):
Params = {}
for ParamCategory in BPatternGraphType.Names:
Params[ParamCategory] = {}
try:
Sequence_NTasksWithWeights = \
AIParseUtils.readIntWithWeightsList(
WLUnit['otherinfo']['Params.Sequence.NTasksWithWeights'],
DefaultWeight = 1.0, ItemSeparator = ','
)
Params['Sequence']['NTasksWithWeights'] = Sequence_NTasksWithWeights
except:
##
print traceback.print_exc()
pass
try:
ParallelSplit_NBranchesWithWeights = \
AIParseUtils.readIntWithWeightsList(
WLUnit['otherinfo']['Params.ParallelSplit.NBranchesWithWeights'],
DefaultWeight = 1.0, ItemSeparator = ','
)
Params['ParallelSplit']['NBranchesWithWeights'] = ParallelSplit_NBranchesWithWeights
except:
##
print traceback.print_exc()
pass
try:
ParallelSplit_MinTasksPerBranchWithWeights = \
AIParseUtils.readIntWithWeightsList(
WLUnit['otherinfo']['Params.ParallelSplit.MinTasksPerBranchWithWeights'],
DefaultWeight = 1.0, ItemSeparator = ','
)
Params['ParallelSplit']['MinTasksPerBranchWithWeights'] = ParallelSplit_MinTasksPerBranchWithWeights
except:
##
print traceback.print_exc()
pass
try:
ParallelSplit_MaxTasksPerBranchWithWeights = \
AIParseUtils.readIntWithWeightsList(
WLUnit['otherinfo']['Params.ParallelSplit.MaxTasksPerBranchWithWeights'],
DefaultWeight = 1.0, ItemSeparator = ','
)
Params['ParallelSplit']['MaxTasksPerBranchWithWeights'] = ParallelSplit_MaxTasksPerBranchWithWeights
except:
##
print traceback.print_exc()
pass
return Params
def readAllParams( WLUnit ):
Params = {}
for ParamCategory in RandomGraphType.Names:
Params[ParamCategory] = {}
try:
TaskSameProb_NTasksWithWeights = \
AIParseUtils.readIntWithWeightsList(
WLUnit['otherinfo']['Params.TaskSameProb.NTasksWithWeights'],
DefaultWeight = 1.0, ItemSeparator = ','
)
Params['TaskSameProb']['NTasksWithWeights'] = TaskSameProb_NTasksWithWeights
except:
##
print traceback.print_exc()
pass
try:
TaskSameProb_pWithWeights = \
AIParseUtils.readFloatWithWeightsList(
WLUnit['otherinfo']['Params.TaskSameProb.pWithWeights'],
DefaultWeight = 1.0, ItemSeparator = ','
)
Params['TaskSameProb']['pWithWeights'] = TaskSameProb_pWithWeights
except:
##
print traceback.print_exc()
pass
try:
TaskSamePred_NTasksWithWeights = \
AIParseUtils.readIntWithWeightsList(
WLUnit['otherinfo']['Params.TaskSamePred.NTasksWithWeights'],
DefaultWeight = 1.0, ItemSeparator = ','
)
Params['TaskSamePred']['NTasksWithWeights'] = TaskSamePred_NTasksWithWeights
except:
##
print traceback.print_exc()
pass
try:
TaskSamePred_NTasksWithWeights = \
AIParseUtils.readIntWithWeightsList(
WLUnit['otherinfo']['Params.TaskSamePred.NConnectWithWeights'],
DefaultWeight = 1.0, ItemSeparator = ','
)
Params['TaskSamePred']['NConnectWithWeights'] = TaskSamePred_NTasksWithWeights
except:
##
print traceback.print_exc()
pass
try:
LayerSameProb_NLayersWithWeights = \
AIParseUtils.readTupleWithWeightsList(
WLUnit['otherinfo']['Params.LayerSameProb.NLayersWithWeights'],
TupleType = AIParseUtils.TupleValues.INT_VALUE,
DefaultWeight = 1.0, ItemSeparator = ';', TupleItemSeparator=','
)
Params['LayerSameProb']['NLayersWithWeights'] = LayerSameProb_NLayersWithWeights
except:
##
print traceback.print_exc()
pass
try:
LayerSameProb_NLayerTasksWithWeights = \
AIParseUtils.readTupleWithWeightsList(
WLUnit['otherinfo']['Params.LayerSameProb.NLayerTasksWithWeights'],
TupleType = AIParseUtils.TupleValues.INT_VALUE,
DefaultWeight = 1.0, ItemSeparator = ';', TupleItemSeparator=','
)
Params['LayerSameProb']['NLayerTasksWithWeights'] = LayerSameProb_NLayerTasksWithWeights
except:
##
print traceback.print_exc()
pass
try:
LayerSameProb_pWithWeights = \
AIParseUtils.readFloatWithWeightsList(
WLUnit['otherinfo']['Params.LayerSameProb.pWithWeights'],
DefaultWeight = 1.0, ItemSeparator = ','
)
Params['LayerSameProb']['pWithWeights'] = LayerSameProb_pWithWeights
except:
##
print traceback.print_exc()
pass
try:
LayerSamePred_NLayersWithWeights = \
AIParseUtils.readTupleWithWeightsList(
WLUnit['otherinfo']['Params.LayerSamePred.NLayersWithWeights'],
TupleType = AIParseUtils.TupleValues.INT_VALUE,
DefaultWeight = 1.0, ItemSeparator = ';', TupleItemSeparator=','
)
Params['LayerSamePred']['NLayersWithWeights'] = LayerSamePred_NLayersWithWeights
except:
##
print traceback.print_exc()
pass
try:
LayerSamePred_NLayerTasksWithWeights = \
AIParseUtils.readTupleWithWeightsList(
WLUnit['otherinfo']['Params.LayerSamePred.NLayerTasksWithWeights'],
TupleType = AIParseUtils.TupleValues.INT_VALUE,
DefaultWeight = 1.0, ItemSeparator = ';', TupleItemSeparator=','
)
Params['LayerSamePred']['NLayerTasksWithWeights'] = LayerSamePred_NLayerTasksWithWeights
except:
##
print traceback.print_exc()
pass
try:
LayerSamePred_NTasksWithWeights = \
AIParseUtils.readIntWithWeightsList(
WLUnit['otherinfo']['Params.LayerSamePred.NConnectWithWeights'],
DefaultWeight = 1.0, ItemSeparator = ','
)
Params['LayerSamePred']['NConnectWithWeights'] = LayerSamePred_NTasksWithWeights
except:
##
print traceback.print_exc()
pass
return Params
def generateRandomStructures(WLOutDir, WLUnitID, WLUnit, SubmitDurationMS):
#--- params read
##print ">>> WLUnit=", WLUnit
try:
NCompositeItems = AIParseUtils.readInt(WLUnit['otherinfo']['NCompositeItems'], 1)
except:
NCompositeItems = 1
try:
GraphTypesWithWeights = \
AIParseUtils.readStringWithWeightsList(
WLUnit['otherinfo']['GraphTypesWithWeights'],
DefaultWeight = 1.0, ItemSeparator = ';'
)
## ValidGraphTypesWithWeights = {}
## for GraphType in GraphTypesWithWeights[AIParseUtils.VALUES]:
## if GraphTypes not in RandomGraphType.Names:
## print "WARNING!", GraphTypes, "is not a valid GraphType. Skipping it."
## else:
## ValidGraphTypesWithWeights[GraphTypes] = GraphTypesWithWeights[AIParseUtils.VALUES][GraphTypes]
## GraphTypesWithWeights[AIParseUtils.VALUES] = ValidGraphTypesWithWeights
except:
print "ERROR! No GraphTypesWithWeights info specified in the OtherInfo field.\n\tQuitting!"
return None
##print ">>> GraphTypesWithWeights=", GraphTypesWithWeights
STGFileNamesList = []
try:
STGFilters = WLUnit['otherinfo']['STGFiles']
STGFilterList = STGFilters.split(',')
for Filter in STGFilterList:
CurrentSTGList = glob.glob(Filter)
for STGFileName in CurrentSTGList:
if STGFileName not in STGFileNamesList:
STGFileNamesList.append(STGFileName)
except:
STGFileNamesList = []
pass
TextGraphFileNamesList = []
try:
TextGraphFilters = WLUnit['otherinfo']['TextGraphFiles']
TextGraphFilterList = TextGraphFilters.split(',')
for Filter in TextGraphFilterList:
CurrentTextGraphList = glob.glob(Filter)
for TextGraphFileName in CurrentTextGraphList:
if TextGraphFileName not in TextGraphFileNamesList:
TextGraphFileNamesList.append(TextGraphFileName)
#print "#### appending: " + TextGraphFileName
except:
TextGraphFileNamesList = []
#print "### TextGraph exception"
#print traceback.print_exc()
pass
##print ">>>", "before Params = readAllParams( WLUnit )"
Params = readAllParams( WLUnit )
##print ">>>", "after Params = readAllParams( WLUnit )"
#--- init
ListOfGraphs = []
#--- generate all composite structures
index = 0
maxIndex = NCompositeItems
while index < maxIndex:
#--- generate one composite structure
GraphID = WLUnitID + ("-%d_graph" % index)
#print("### Graph ID initial: " + GraphID)
#print ("### WLUnit ID initial: " + WLUnitID)
GraphType = AIRandomUtils.getRandomWeightedListElement(
GraphTypesWithWeights,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
GraphType = RandomGraphType.Names[GraphType]
#print ">>>", "Selected", GraphType, "from", GraphTypesWithWeights
GraphName = getRandomGraphTypeName(GraphType)
GraphData = {}
if GraphType == RandomGraphType.FromSTG:
pass
elif GraphType == RandomGraphType.TaskSameProb:
try:
GraphData['NTasks'] = \
AIRandomUtils.getRandomWeightedListElement(
Params['TaskSameProb']['NTasksWithWeights'],
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
GraphData['p'] = \
AIRandomUtils.getRandomWeightedListElement(
Params['TaskSameProb']['pWithWeights'],
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
except Exception, e:
print "Could not find a definition for", getRandomGraphTypeName(GraphType), "parameter", e
continue
GraphName = GraphName + ' NTasks=' + str(GraphData['NTasks']) + ' p=' + str(GraphData['p'])
elif GraphType == RandomGraphType.TaskSamePred:
try:
GraphData['NTasks'] = \
AIRandomUtils.getRandomWeightedListElement(
Params['TaskSamePred']['NTasksWithWeights'],
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
GraphData['NConnect'] = \
AIRandomUtils.getRandomWeightedListElement(
Params['TaskSamePred']['NConnectWithWeights'],
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
except Exception, e:
print "Could not find a definition for", getRandomGraphTypeName(GraphType), "parameter", e
continue
def generateWorkloadUnit( UnitDir, UnitID, WLUnit, SubmitDurationMS, bGenerateRandom = 1 ):
"""
Out:
UnitsDic
o A dictionary with keys 'info' and 'jobs'.
Notes:
o UnitsDic['info'] contains a dictionary of jobs info,
indexed with an integer counter
o Each job info contains at least the keys
name, description, jdf, submitCommand, runTime
o UnitsDic['jobs'] contains a dictionary of jobs data,
indexed with an integer counter
o Each job data contains the list of components of that job
o Each component in the list is a dictionary with at least the
following fields: executable, stdout, stderr, name, description,
directory, maxWallTime, arguments, env, stagein, stageout
See also:
utils/WLDocHandlers.WLSubmitJobKeys
this_file.SSERComponent.generateComponent
"""
global SSER_LastRunTime
SSERComponent.SSER_Exe = "/home/antoniou/Synthetic_Apps/sseriot" #os.path.join( "/tmp", "ssert" )
SSERComponent.SSER_ParKSizes = [ "0", "32", "128", "512", "1024" ]
SSERComponent.SSER_ParKSuperSizes = [ "0", "16", "32", "64", "128", "256" ]
SSERComponent.SSER_ParSupersteps = [ "1", "2", "5", "10", "20", "50", "100" ]
SSERComponent.SSER_ParMemoryKItems = [ "10", "25", "50", "100", "250", "500", "1000", "5000" ]
SSERComponent.SSER_ParMemoryElementsPerItem = [ "3", "4", "10", "100", "500", "1000" ]
SSERComponent.SSER_ParComputationPerMemoryItem = [ "2", "10", "100", "1000" ]
SSERComponent.SSER_RunTimeInMinutes = [ "2", "5", "10", "15" ]
if 'otherinfo' in WLUnit.keys():
try:
OneString = WLUnit['otherinfo']['Exe']
SSERComponent.SSER_Exe = OneString
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParKSizes'],
ItemSeparator = ',' )
SSERComponent.SSER_ParKSizes = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParKSuperSizes'],
ItemSeparator = ',' )
SSERComponent.SSER_ParKSuperSizes = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParSupersteps'],
ItemSeparator = ',' )
SSERComponent.SSER_ParSupersteps = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParMemoryKItems'],
ItemSeparator = ',' )
SSERComponent.SSER_ParMemoryKItems = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParMemoryElementsPerItem'],
ItemSeparator = ',' )
SSERComponent.SSER_ParMemoryElementsPerItem = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParComputationPerMemoryItem'],
ItemSeparator = ',' )
SSERComponent.SSER_ParComputationPerMemoryItem = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['RunTimeInMinutes'],
ItemSeparator = ',' )
SSERComponent.SSER_RunTimeInMinutes = OneIntList
except KeyError:
pass
UnitsDic = {}
UnitsDic['info'] = {}
UnitsDic['jobs'] = {}
#-- init start time
try:
StartAt = AIParseUtils.readInt(WLUnit['otherinfo']['StartAt'], DefaultValue = 0)
StartAt = StartAt * 1000 # the time is given is seconds
except KeyError:
StartAt = 0
SSER_LastRunTime = StartAt
if 'FirstJobIndex' in WLUnit:
JobIndex = int(WLUnit['FirstJobIndex'])
else:
JobIndex = 0
if bGenerateRandom > 0 :
index = 0
maxIndex = WLUnit['multiplicity']
while index < maxIndex:
UnitsDic['info'][index] = \
generateJobInfo( UnitDir, UnitID, JobIndex, \
WLUnit, SubmitDurationMS )
UnitsDic['jobs'][index] = \
generateJobComponents( UnitDir, UnitID, JobIndex, WLUnit, \
bGenerateRandom )
index = index + 1
JobIndex = JobIndex + 1
WLUnit['generatedjobs'] = index
else:
index = 0
for Size in SSERComponent.SSER_ParKSuperSizes:
for N in SSERComponent.SSER_ParSupersteps:
for M in SSERComponent.SSER_ParMemoryKItems:
for S in SSERComponent.SSER_ParMemoryElementsPerItem:
for C in SSERComponent.SSER_ParComputationPerMemoryItem:
for MaxWallTime in SSERComponent.SSER_RunTimeInSeconds:
UnitsDic['info'][index] = \
generateJobInfo( UnitDir, UnitID, JobIndex, \
WLUnit, SubmitDurationMS )
UnitsDic['jobs'][index] = \
generateJobComponents( UnitDir, UnitID, JobIndex, WLUnit, \
bGenerateRandom, Size, N, M, S, C, MaxWallTime)
index = index + 1
JobIndex = JobIndex + 1
#-- set generated jobs
WLUnit['generatedjobs'] = index
return UnitsDic
def generateWorkload( UnitDir, UnitID, WLUnit, SubmitDurationMS, bGenerateRandom = 1 ):
global SWF_LastRunTime
SWFConstsInstance = SWFConsts()
CurrentSWFParams = readParams( WLUnit )
for Filter in CurrentSWFParams.SWF.Filters:
print 'Filter', Filter, '->', CurrentSWFParams.SWF.Filters[Filter]
JobsList, UNIXStartTime = readSWFFile( CurrentSWFParams.SWF.File, CurrentSWFParams )
print "After filtering out,", len(JobsList), "jobs remained."
##for Job in JobsList:
## print Job
UnitsDic = {}
UnitsDic['info'] = {}
UnitsDic['jobs'] = {}
#-- init start time
try:
StartAt = AIParseUtils.readInt(WLUnit['otherinfo']['StartAt'], DefaultValue = 0)
StartAt = StartAt * 1000 # the time is given is seconds
except KeyError:
StartAt = 0
SWF_LastRunTime = StartAt
WLUnitMultiplicity = int(WLUnit['multiplicity'])
FirstJob = None
JobIndex = 0
NJobsInList = len(JobsList)
for GroupID in xrange(WLUnitMultiplicity):
for JobID in xrange(NJobsInList):
#JobIndex = JobID + GroupID * NJobsInList
#--- get trace data
Job = JobsList[JobID]
##print Job
if FirstJob is None:
FirstJob = Job
# 2. select the number of CPU for this job
#TODO: repeat NCPUs selection until App can run on that many CPUs
NCPUs = Job[SWFConstsInstance.NCPUs]
# 3. select the number of components and try to create subcomponents,
# until successfully matching restrictions
WhileCondition = 1
NTries = 250
while WhileCondition != 0 and NTries > 0:
NTries = NTries - 1
NComponents = AIRandomUtils.getRandomWeightedListElement(
CurrentSWFParams.NComponentsWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
if NComponents > NCPUs:
continue # impossible match, just retry
NCPUsPerComponent = NCPUs / NComponents
# ensure restrictions are met
if NCPUsPerComponent < CurrentSWFParams.MinComponentSize or \
NCPUsPerComponent > CurrentSWFParams.MaxComponentSize:
continue
NRemainingComponents = NCPUs - NCPUsPerComponent * NComponents
if NRemainingComponents > 0 and CurrentSWFParams.EqualCPUsPerComponent == 1:
continue
# >= -> including the extra CPU for some components
if NRemainingComponents > 0 and NCPUsPerComponent == CurrentSWFParams.MaxComponentSize:
continue
SiteType = AIRandomUtils.getRandomWeightedListElement(
CurrentSWFParams.SiteTypesWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
PreComponentsList = []
for ComponentID in xrange(NComponents):
# cpu count
ToAssignHere = NCPUsPerComponent
if ComponentID < NRemainingComponents: # assign the extra CPUs, round-robin
ToAssignHere = ToAssignHere + 1
# site location
if SiteType == 'ordered':
Site = AIRandomUtils.getRandomWeightedListElement(
CurrentSWFParams.SitesWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
else:
Site = '*'
# create pre-component information
PreComponent = {}
PreComponent['id'] = ComponentID
PreComponent['count'] = ToAssignHere
PreComponent['location'] = Site
# add pre-component
PreComponentsList.append(PreComponent)
WhileCondition = 0
# skip errors
if WhileCondition != 0:
continue
#--- generate job
Application = CurrentSWFParams.Application
Submitter = CurrentSWFParams.Submitter
ApplicationBinaryFullPath = os.path.join(CurrentSWFParams.AppBaseDir, CurrentSWFParams.Application)
SWFConstsInstace = SWFConsts()
# convert the start time from ms to s
JobStartTime = 1000 * (Job[SWFConstsInstace.SubmitTime] - FirstJob[SWFConstsInstace.SubmitTime])
if JobStartTime < 0:
print "Flurry! JobStartTime < 0"
print "FirstJob=", FirstJob
print "Job=", Job
#-- generate job: ID
CurrentUnitID = "%s-%d-%d" % (UnitID, GroupID, JobIndex % NJobsInList)
#-- generate job: index
UnitsDic['info'][JobIndex] = \
generateJobInfo( UnitDir, CurrentUnitID, JobIndex, Application, \
WLUnit, SubmitDurationMS, Submitter, \
CurrentSWFParams.UseArrivalTimeDistribution, JobStartTime )
#-- generate job: components
UnitsDic['jobs'][JobIndex] = generateJobComponents(
UnitDir, CurrentUnitID, JobIndex, WLUnit,
PreComponentsList, ApplicationBinaryFullPath, \
NCPUs, Job, CurrentSWFParams.MaxWallTime )
#-- be more descriptive
UnitsDic['info'][JobIndex]['description'] = \
UnitsDic['info'][JobIndex]['description'] + \
" Components: %d. NCPUs: %d." % (len(PreComponentsList), NCPUs)
#--- done generating
if UnitsDic['jobs'][JobIndex] == None:
NJobs = NJobs - 1 #-- don't ack job, but make sure total no of jobs is decreased
else:
print "SWF generator: Generated a job of type", Application,
print "with", len(PreComponentsList), "components/",
print NCPUs, "CPUs, in", UnitsDic['info'][JobIndex]['jdf']
JobIndex = JobIndex + 1
# add the delay between separate units
SWF_LastRunTime = SWF_LastRunTime + CurrentSWFParams.DelayBetweenUnits
print "Generated", JobIndex, "jobs."
WLUnit['generatedjobs'] = JobIndex
#raise Exception, "SWF generator: Not implemented yet"
return UnitsDic
def readParams( ParamsDic ):
""" read params, return a SWFParams class, with new fields """
CurrentSWFParams = SWFParams()
CurrentSWFParams.SWF = SWFOnlyParams()
if 'otherinfo' in ParamsDic.keys():
SWFConstsInstance = SWFConsts()
try:
for key in ParamsDic['otherinfo']:
OneList = key.split('.')
if OneList[0] == 'SWF':
if OneList[1] == 'File':
CurrentSWFParams.SWF.File = ParamsDic['otherinfo'][key]
elif OneList[1] == 'Scale': # SWF.Scale.SubmitTime=1:1
KeyID = SWFConstsInstance.getID(OneList[2])
if (KeyID is not None) and (KeyID >= 0):
# save: ['SubmitTime']: (1,1)
ScaleIn, ScaleOut = ParamsDic['otherinfo'][key].split(':')
ScaleIn = float(ScaleIn)
ScaleOut = float(ScaleOut)
CurrentSWFParams.SWF.Scale[OneList[2]] = (ScaleIn, ScaleOut)
elif OneList[1] == 'Filter':
KeyID = SWFConstsInstance.getID(OneList[2])
##print 'Filter', OneList[2], "KeyID=", KeyID
if (KeyID is not None) and (KeyID >= 0):
#SWF.Filter.JobNumber.Min=31
# save: ['JobNumber']: ('Min',31)
if OneList[2] not in CurrentSWFParams.SWF.Filters:
CurrentSWFParams.SWF.Filters[OneList[2]] = []
CurrentSWFParams.SWF.Filters[OneList[2]].append((OneList[3], int(ParamsDic['otherinfo'][key])))
else:
# special cases
##print "Special case filter", OneList[2]
# SWF.Filter.Delta.SubmitTime.Hours.Max=3
# save: ['Delta']['SubmitTime']['Hours']('Max',31)
if OneList[2] == 'Delta':
if 'Delta' not in CurrentSWFParams.SWF.Filters:
CurrentSWFParams.SWF.Filters['Delta'] = {}
KeyID = SWFConstsInstance.getID(OneList[3])
if KeyID and KeyID >= 0:
if OneList[3] not in CurrentSWFParams.SWF.Filters['Delta']:
CurrentSWFParams.SWF.Filters['Delta'][OneList[3]] = {}
CurrentSWFParams.SWF.Filters['Delta'][OneList[3]][OneList[4]] = (OneList[5], int(ParamsDic['otherinfo'][key]))
# SWF.Filter.NJobs.Max=1000
# save: ['NJobs']: ('Max',1000)
if OneList[2] == 'NJobs':
CurrentSWFParams.SWF.Filters['NJobs'] = (OneList[3], int(ParamsDic['otherinfo'][key]))
except Exception, e:
print "Exception", e
print traceback.print_exc()
raise Exception, "SWF generator: could not decode " + str(key) + '.'
if CurrentSWFParams.SWF.File is None:
raise Exception, "SWF generator: No SWF.File field in OtherInfo (cannot read an unknown file)."
# UseArrivalTimeDistribution=false
try:
CurrentSWFParams.UseArrivalTimeDistribution = AIParseUtils.readBoolean(ParamsDic['otherinfo']['UseArrivalTimeDistribution'])
except KeyError:
CurrentSWFParams.UseArrivalTimeDistribution = 0
#Application=swf
#AppBaseDir=/tmp
try:
CurrentSWFParams.AppBaseDir = ParamsDic['otherinfo']['AppBaseDir']
except KeyError:
CurrentSWFParams.AppBaseDir = '/tmp'
#raise Exception, "SMPI1 generator: No AppBaseDir info specified in the OtherInfo field."
print "WARNING! SWF generator: No AppBaseDir info specified in the OtherInfo field.", "Assuming default:", AppBaseDir
try:
CurrentSWFParams.Application = ParamsDic['otherinfo']['Application']
except KeyError:
CurrentSWFParams.Application = 'swf'
#raise Exception, "SMPI1 generator: No AppBaseDir info specified in the OtherInfo field."
print "WARNING! SWF generator: No Application info specified in the OtherInfo field.", "Assuming default:", Application
try:
CurrentSWFParams.Submitter = ParamsDic['otherinfo']['Submitter']
except KeyError:
CurrentSWFParams.Submitter = 'krunner -l DEBUG -g -e -o -f ${JDF}'
print "WARNING! SWF generator: No Submitter info specified in the OtherInfo field.", "Assuming default", Submitter
try:
CurrentSWFParams.NComponentsWithWeightsDic = \
AIParseUtils.readIntWithWeightsList(
ParamsDic['otherinfo']['NComponentsWithWeights'],
DefaultWeight = 1.0
)
# NComponentsWithWeightsDic['Values']; NComponentsWithWeightsDic['TotalWeight']
except KeyError:
raise Exception, "SWF generator: Invalid NComponentsWithWeights info specified in the OtherInfo field."
##print ">>>>>> NComponentsWithWeightsDic", NComponentsWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
try:
CurrentSWFParams.MinComponentSize = AIParseUtils.readInt(ParamsDic['otherinfo']['MinComponentSize'], DefaultValue = 0)
except KeyError:
CurrentSWFParams.MinComponentSize = 1
print "WARNING! SWF generator: No MinComponentSize info specified in the OtherInfo field.", "Assuming default", CurrentSWFParams.MinComponentSize
#raise Exception, "SWF generator: No MinComponentSize info specified in the OtherInfo field."
try:
CurrentSWFParams.DelayBetweenUnits = AIParseUtils.readInt(ParamsDic['otherinfo']['DelayBetweenUnits'], DefaultValue = 0)
except KeyError:
CurrentSWFParams.DelayBetweenUnits = 1
print "WARNING! SWF generator: No DelayBetweenUnits info specified in the OtherInfo field.", "Assuming default", CurrentSWFParams.DelayBetweenUnits
try:
CurrentSWFParams.MaxWallTime = AIParseUtils.readInt(ParamsDic['otherinfo']['MaxWallTime'], DefaultValue = 1000000)
except KeyError:
CurrentSWFParams.MaxWallTime = 1
print "WARNING! SWF generator: No DelayBetweenUnits info specified in the OtherInfo field.", "Assuming default", CurrentSWFParams.MaxWallTime
try:
CurrentSWFParams.MaxComponentSize = AIParseUtils.readInt(ParamsDic['otherinfo']['MaxComponentSize'], DefaultValue = 0)
except KeyError:
CurrentSWFParams.MaxComponentSize = 10000000
print "WARNING! SWF generator: No MaxComponentSize info specified in the OtherInfo field.", "Assuming default", CurrentSWFParams.MaxComponentSize
#raise Exception, "SWF generator: No MaxComponentSize info specified in the OtherInfo field."
try:
CurrentSWFParams.EqualCPUsPerComponent = AIParseUtils.readBoolean(ParamsDic['otherinfo']['SWFAppsDir'])
except KeyError:
CurrentSWFParams.EqualCPUsPerComponent = 0
#raise Exception, "SWF generator: No EqualCPUsPerComponent info specified in the OtherInfo field. Setting to default " + AIParseUtils.IntToBooleanDic[EqualCPUsPerComponent] + '.'
try:
CurrentSWFParams.SiteTypesWithWeightsDic = \
AIParseUtils.readStringWithWeightsList(
ParamsDic['otherinfo']['SiteTypesWithWeights'],
DefaultWeight = 1.0
)
except KeyError:
CurrentSWFParams.SiteTypesWithWeights = {AIParseUtils.VALUES:[], AIParseUtils.TOTAL_WEIGHT:0.0 }
print "WARNING! SWF generator: No SiteTypesWithWeights info specified in the OtherInfo field.\n\tSetting to default..."
##print ">>>>>> SiteTypesWithWeightsDic", SiteTypesWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
try:
CurrentSWFParams.SitesWithWeightsDic = \
AIParseUtils.readStringWithWeightsList(
ParamsDic['otherinfo']['SitesWithWeights'],
DefaultWeight = 1.0
)
except KeyError:
CurrentSWFParams.SitesWithWeightsDic = {AIParseUtils.VALUES:[], AIParseUtils.TOTAL_WEIGHT:0.0 }
print "WARNING! SWF generator: No SitesWithWeights info specified in the OtherInfo field.\n\tSetting to default..."
for name in JDFGeneratorNames:
ResManagerName = name.replace('-jdf', '')
DirName = "jdfs-%s" % ResManagerName
OutDirName = WLOutDir.replace('jdfs', DirName)
if not os.path.exists(OutDirName):
os.makedirs(OutDirName)
except OSError, e:
print "Cannot create output directory for unit", WLUnitID, "...skipping unit"
print '\tOS returned:', e
continue
if 'RandomWorkload' in WLUnit['otherinfo'].keys():
try:
##print "Getting random info from", WLUnit['otherinfo']['RandomWorkload']
bGenerateRandom = AIParseUtils.readBoolean(WLUnit['otherinfo']['RandomWorkload'], 1)
except Exception, e:
print "Wrong value for RandomWorkload", "("+WLUnit['otherinfo']['RandomWorkload'].lower()+"). Expected true/false. Setting to default (true)"
bGenerateRandom = 1
else:
print "No value for RandomWorkload", "Setting to default (true)."
bGenerateRandom = 1
WLUnit['arrivaltimefunc'] = ArrivalTimeRandom.randomVariable
#--- get appropriate generator function
#--- v1.1: composite
# composite?
if WLUnit['composite'] == 1:
bGenerateRandom = 1 ### composite apps are always randomly generated
def generateWorkloadUnit( UnitDir, UnitID, WLUnit, SubmitDurationMS, bGenerateRandom = 1 ):
"""
Out:
UnitsDic
o A dictionary with keys 'info' and 'jobs'.
Notes:
o UnitsDic['info'] contains a dictionary of jobs info,
indexed with an integer counter
o Each job info contains at least the keys
name, description, jdf, submitCommand, runTime
o UnitsDic['jobs'] contains a dictionary of jobs data,
indexed with an integer counter
o Each job data contains the list of components of that job
o Each component in the list is a dictionary with at least the
following fields: executable, stdout, stderr, name, description,
directory, maxWallTime, arguments, env, stagein, stageout
See also:
utils/WLDocHandlers.WLSubmitJobKeys
this_file.SMPI1Component.generateComponent
"""
global SMPI1_LastRunTime
SMPI1Component.SMPI1_Exe = os.path.join( "/tmp", "smpi1t" )
SMPI1Component.SMPI1_ParKSizes = [ "0", "32", "128", "512", "1024" ]
SMPI1Component.SMPI1_ParKSuperSizes = [ "0", "16", "32", "64", "128", "256" ]
SMPI1Component.SMPI1_ParSupersteps = [ "1", "2", "5", "10", "20", "50", "100" ]
SMPI1Component.SMPI1_ParMemoryKItems = [ "10", "25", "50", "100", "250", "500", "1000", "5000" ]
SMPI1Component.SMPI1_ParMemoryElementsPerItem = [ "3", "4", "10", "100", "500", "1000" ]
SMPI1Component.SMPI1_ParXChangeElementsPerStep = [ "100", "500", "1000", "10000", "50000" ]
SMPI1Component.SMPI1_ParComputationPerMemoryItem = [ "2", "10", "100", "1000" ]
SMPI1Component.SMPI1_RunTimeInMinutes = [ "2", "5", "10", "15" ]
if 'otherinfo' in WLUnit.keys():
try:
OneString = WLUnit['otherinfo']['Exe']
SMPI1Component.SMPI1_Exe = OneString
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParKSizes'],
ItemSeparator = ',' )
SMPI1Component.SMPI1_ParKSizes = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParKSuperSizes'],
ItemSeparator = ',' )
SMPI1Component.SMPI1_ParKSuperSizes = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParSupersteps'],
ItemSeparator = ',' )
SMPI1Component.SMPI1_ParSupersteps = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParMemoryKItems'],
ItemSeparator = ',' )
SMPI1Component.SMPI1_ParMemoryKItems = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParMemoryElementsPerItem'],
ItemSeparator = ',' )
SMPI1Component.SMPI1_ParMemoryElementsPerItem = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParXChangeElementsPerStep'],
ItemSeparator = ',' )
SMPI1Component.SMPI1_ParXChangeElementsPerStep = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParComputationPerMemoryItem'],
ItemSeparator = ',' )
SMPI1Component.SMPI1_ParComputationPerMemoryItem = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['RunTimeInMinutes'],
ItemSeparator = ',' )
SMPI1Component.SMPI1_RunTimeInMinutes = OneIntList
except KeyError:
pass
try:
NComponentsWithWeightsDic = \
AIParseUtils.readIntWithWeightsList(
WLUnit['otherinfo']['NComponentsWithWeights'],
DefaultWeight = 1.0
)
# NComponentsWithWeightsDic['Values']; NComponentsWithWeightsDic['TotalWeight']
except KeyError:
raise Exception, "SMPI1 generator: Invalid NComponentsWithWeights info specified in the OtherInfo field."
##print ">>>>>> NComponentsWithWeightsDic", NComponentsWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
try:
TotalCPUsWithWeightsDic = \
AIParseUtils.readIntWithWeightsList(
WLUnit['otherinfo']['TotalCPUsWithWeights'],
DefaultWeight = 1.0
)
except KeyError:
raise Exception, "SMPI1 generator: Invalid TotalCPUsWithWeights info specified in the OtherInfo field."
##print ">>>>>> TotalCPUsWithWeightsDic", TotalCPUsWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
try:
MinComponentSize = AIParseUtils.readInt(WLUnit['otherinfo']['MinComponentSize'], DefaultValue = 0)
except KeyError:
MinComponentSize = 1
#raise Exception, "SMPI1 generator: No MinComponentSize info specified in the OtherInfo field."
try:
MaxComponentSize = AIParseUtils.readInt(WLUnit['otherinfo']['MaxComponentSize'], DefaultValue = 0)
except KeyError:
MaxComponentSize = 10000000
#raise Exception, "SMPI1 generator: No MaxComponentSize info specified in the OtherInfo field."
try:
EqualCPUsPerComponent = AIParseUtils.readBoolean(WLUnit['otherinfo']['SMPI1AppsDir'])
except KeyError:
EqualCPUsPerComponent = 0
#raise Exception, "SMPI1 generator: No EqualCPUsPerComponent info specified in the OtherInfo field. Setting to default " + AIParseUtils.IntToBooleanDic[EqualCPUsPerComponent] + '.'
try:
AppBaseDir = WLUnit['otherinfo']['AppBaseDir']
except KeyError:
AppBaseDir = '/tmp'
#raise Exception, "SMPI1 generator: No AppBaseDir info specified in the OtherInfo field."
print "WARNING! SMPI1 generator: No AppBaseDir info specified in the OtherInfo field.", "Assuming default:", AppBaseDir
try:
Application = WLUnit['otherinfo']['Application']
except KeyError:
Application = 'smpi1t-gm'
#raise Exception, "SMPI1 generator: No AppBaseDir info specified in the OtherInfo field."
print "WARNING! SMPI1 generator: No Application info specified in the OtherInfo field.", "Assuming default:", Application
try:
Submitter = WLUnit['otherinfo']['Submitter']
except KeyError:
Submitter = 'krunner -l DEBUG -g -e -o -f ${JDF}'
print "WARNING! SMPI1 generator: No Submitter info specified in the OtherInfo field.", "Assuming default", Submitter
try:
SiteTypesWithWeightsDic = \
AIParseUtils.readStringWithWeightsList(
WLUnit['otherinfo']['SiteTypesWithWeights'],
DefaultWeight = 1.0
)
except KeyError:
SiteTypesWithWeights = {AIParseUtils.VALUES:[], AIParseUtils.TOTAL_WEIGHT:0.0 }
print "WARNING! SMPI1 generator: No SiteTypesWithWeights info specified in the OtherInfo field.\n\tSetting to default", SitesList
##print ">>>>>> SiteTypesWithWeightsDic", SiteTypesWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
try:
SitesWithWeightsDic = \
AIParseUtils.readStringWithWeightsList(
WLUnit['otherinfo']['SitesWithWeights'],
DefaultWeight = 1.0
)
except KeyError:
SitesWithWeightsDic = {AIParseUtils.VALUES:[], AIParseUtils.TOTAL_WEIGHT:0.0 }
print "WARNING! SMPI1 generator: No SitesWithWeights info specified in the OtherInfo field.\n\tSetting to default", SitesList
##print ">>>>>> SitesWithWeightsDic", SitesWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
else:
raise Exception, "SMPI1 generator: No OtherInfo data! Expected at least ComponentSize,NJobs,NComponents fields."
UnitsDic = {}
UnitsDic['info'] = {}
UnitsDic['jobs'] = {}
#-- init start time
try:
StartAt = AIParseUtils.readInt(WLUnit['otherinfo']['StartAt'], DefaultValue = 0)
StartAt = StartAt * 1000 # the time is given is seconds
except KeyError:
StartAt = 0
SMPI1_LastRunTime = StartAt
if 'FirstJobIndex' in WLUnit:
JobIndex = int(WLUnit['FirstJobIndex'])
else:
JobIndex = 0
WLUnitMultiplicity = int(WLUnit['multiplicity'])
if bGenerateRandom > 0 :
index = 0
while index < WLUnitMultiplicity:
# 1. generate a random site type (unordered or ordered)
SiteType = AIRandomUtils.getRandomWeightedListElement(
SiteTypesWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
# 2. select the number of CPU for this job
#TODO: repeat NCPUs selection until App can run on that many CPUs
NCPUs = AIRandomUtils.getRandomWeightedListElement(
TotalCPUsWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
# 3. select the number of components and try to create subcomponents,
# until successfully matching restrictions
WhileCondition = 1
NTries = 100
while WhileCondition != 0 and NTries > 0:
NTries = NTries - 1
NComponents = AIRandomUtils.getRandomWeightedListElement(
NComponentsWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
if NComponents > NCPUs:
continue # impossible match, just retry
NCPUsPerComponent = NCPUs / NComponents
# ensure restrictions are met
if NCPUsPerComponent < MinComponentSize or NCPUsPerComponent > MaxComponentSize:
continue
NRemainingComponents = NCPUs - NCPUsPerComponent * NComponents
if NRemainingComponents > 0 and EqualCPUsPerComponent == 1:
continue
# >= -> including the extra CPU for some components
if NRemainingComponents > 0 and NCPUsPerComponent == MaxComponentSize:
continue
PreComponentsList = []
for ComponentID in xrange(NComponents):
# cpu count
ToAssignHere = NCPUsPerComponent
if ComponentID < NRemainingComponents: # assign the extra CPUs, round-robin
ToAssignHere = ToAssignHere + 1
# site location
if SiteType == 'ordered':
Site = AIRandomUtils.getRandomWeightedListElement(
SitesWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
else:
Site = '*'
# create pre-component information
PreComponent = {}
PreComponent['id'] = ComponentID
PreComponent['count'] = ToAssignHere
PreComponent['location'] = Site
# add pre-component
PreComponentsList.append(PreComponent)
WhileCondition = 0
if WhileCondition == 1: # failed to generate in 100 tries
continue # try again for the same job
#--- generate rnd job
#-- generate job info
UnitsDic['info'][index] = \
generateJobInfo( UnitDir, UnitID, JobIndex, \
WLUnit, SubmitDurationMS, Submitter )
#-- generate job components
UnitsDic['jobs'][index] = \
generateJobComponents( UnitDir, UnitID, JobIndex, WLUnit, PreComponentsList, \
Application, AppBaseDir, \
bGenerateRandom )
index = index + 1
JobIndex = JobIndex + 1
WLUnit['generatedjobs'] = index
else:
index = 0
for Size in SMPI1Component.SMPI1_ParKSuperSizes:
for N in SMPI1Component.SMPI1_ParSupersteps:
for M in SMPI1Component.SMPI1_ParMemoryKItems:
for S in SMPI1Component.SMPI1_ParMemoryElementsPerItem:
for X in SMPI1Component.SMPI1_ParXChangeElementsPerStep:
for C in SMPI1Component.SMPI1_ParComputationPerMemoryItem:
raise Exception, "ERROR! SMPI1 is not finished!"
#--- generate wall time once for all components of this job
MaxWallTime = AIRandomUtils.getRandomListElement( SMPI1Component.SMPI1_RunTimeInSeconds )
#-- generate a random site type (unordered, ordered, ...)
SiteType = AIRandomUtils.getRandomWeightedListElement(
SiteTypesWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
#-- generate job info
UnitsDic['info'][index] = \
generateJobInfo( UnitDir, UnitID, JobIndex, \
WLUnit, SubmitDurationMS )
#-- generate job components
UnitsDic['jobs'][index] = \
generateJobComponents( UnitDir, UnitID, JobIndex, WLUnit, \
bGenerateRandom, SiteType, Size, \
N, M, S, X, C, MaxWallTime )
index = index + 1
JobIndex = JobIndex + 1
#-- set generated jobs
WLUnit['generatedjobs'] = index
return UnitsDic
def generateWorkload( UnitDir, UnitID, WLUnit, SubmitDurationMS, bGenerateRandom = 1 ):
global IBIS_LastRunTime
#print WLUnit['otherinfo']
#{'RandomWorkload': 'true', 'NComponents': '1-8', 'NJobs': '50', 'ComponentSize':'4-8'}
if bGenerateRandom > 0:
print "Random workload"
else:
print "Full workload"
if 'otherinfo' in WLUnit.keys():
try:
##print ">>>>>> NJobs"
NJobs = AIParseUtils.readInt(WLUnit['otherinfo']['NJobs'], DefaultValue = 0)
except KeyError:
NJobs = 0
if bGenerateRandom > 0: #-- NJobs required if random, optional otherwise
raise Exception, "IBIS generator: No NJobs info specified in the OtherInfo field."
try:
NComponentsWithWeightsDic = \
AIParseUtils.readIntWithWeightsList(
WLUnit['otherinfo']['NComponentsWithWeights'],
DefaultWeight = 1.0
)
# NComponentsWithWeightsDic['Values']; NComponentsWithWeightsDic['TotalWeight']
except KeyError:
raise Exception, "IBIS generator: Invalid NComponentsWithWeights info specified in the OtherInfo field."
##print ">>>>>> NComponentsWithWeightsDic", NComponentsWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
try:
TotalCPUsWithWeightsDic = \
AIParseUtils.readIntWithWeightsList(
WLUnit['otherinfo']['TotalCPUsWithWeights'],
DefaultWeight = 1.0
)
except KeyError:
raise Exception, "IBIS generator: Invalid TotalCPUsWithWeights info specified in the OtherInfo field."
##print ">>>>>> TotalCPUsWithWeightsDic", TotalCPUsWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
try:
MinComponentSize = AIParseUtils.readInt(WLUnit['otherinfo']['MinComponentSize'], DefaultValue = 0)
except KeyError:
MinComponentSize = 1
#raise Exception, "IBIS generator: No MinComponentSize info specified in the OtherInfo field."
try:
MaxComponentSize = AIParseUtils.readInt(WLUnit['otherinfo']['MaxComponentSize'], DefaultValue = 0)
except KeyError:
MaxComponentSize = 10000000
#raise Exception, "IBIS generator: No MaxComponentSize info specified in the OtherInfo field."
try:
EqualCPUsPerComponent = AIParseUtils.readBoolean(WLUnit['otherinfo']['IbisAppsDir'])
except KeyError:
EqualCPUsPerComponent = 0
#raise Exception, "IBIS generator: No EqualCPUsPerComponent info specified in the OtherInfo field. Setting to default " + AIParseUtils.IntToBooleanDic[EqualCPUsPerComponent] + '.'
try:
IbisAppsDir = WLUnit['otherinfo']['IbisAppsDir']
except KeyError:
raise Exception, "IBIS generator: No IbisAppsDir info specified in the OtherInfo field."
try:
IbisLibDir = WLUnit['otherinfo']['IbisLibDir']
except KeyError:
raise Exception, "IBIS generator: No IbisLibDir info specified in the OtherInfo field."
try:
JavaHomeDir = WLUnit['otherinfo']['JavaHomeDir']
except KeyError:
raise Exception, "IBIS generator: No JavaHomeDir info specified in the OtherInfo field."
try:
AppsWithWeightsDic = \
AIParseUtils.readStringWithWeightsList(
WLUnit['otherinfo']['AppsWithWeights'],
DefaultWeight = 1.0
)
except KeyError:
raise Exception, "IBIS generator: Invalid AppsWithWeights info specified in the OtherInfo field."
###print ">>>>>> AppsWithWeightsDic", AppsWithWeightsDic#[AIParseUtils.TOTAL_WEIGHT]
try:
Submitter = WLUnit['otherinfo']['Submitter']
except KeyError:
Submitter = 'grunner'
print "WARNING! IBIS generator: No Submitter info specified in the OtherInfo field.\n\tSetting to default", Submitter
try:
SiteTypesWithWeightsDic = \
AIParseUtils.readStringWithWeightsList(
WLUnit['otherinfo']['SiteTypesWithWeights'],
DefaultWeight = 1.0
)
except KeyError:
SiteTypesWithWeights = {AIParseUtils.VALUES:[], AIParseUtils.TOTAL_WEIGHT:0.0 }
print "WARNING! IBIS generator: No SiteTypesWithWeights info specified in the OtherInfo field.\n\tSetting to default", SitesList
##print ">>>>>> SiteTypesWithWeightsDic", SiteTypesWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
try:
SitesWithWeightsDic = \
AIParseUtils.readStringWithWeightsList(
WLUnit['otherinfo']['SitesWithWeights'],
DefaultWeight = 1.0
)
except KeyError:
SitesWithWeightsDic = {AIParseUtils.VALUES:[], AIParseUtils.TOTAL_WEIGHT:0.0 }
print "WARNING! IBIS generator: No SitesWithWeights info specified in the OtherInfo field.\n\tSetting to default", SitesList
##print ">>>>>> SitesWithWeightsDic", SitesWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
try:
JavaMaxMemoryLimitMB = AIParseUtils.readInt(WLUnit['otherinfo']['JavaMaxMemoryLimitMB'], 10000)
except KeyError:
JavaMaxMemoryLimitMB = 10000
print "WARNING! IBIS generator: No JavaMaxMemoryLimitMB info specified in the OtherInfo field.\n\tSetting to default", JavaMaxMemoryLimitMB, "MB"
try:
SelectionMethodName = WLUnit['otherinfo']['SelectionMethod']
except KeyError:
SelectionMethodName = 'R'
raise Exception, "IBIS generator: No SelectionMethod info specified in the OtherInfo field."
else:
raise Exception, "IBIS generator: No OtherInfo data! Expected at least ComponentSize,NJobs,NComponents fields."
UnitsDic = {}
UnitsDic['info'] = {}
UnitsDic['jobs'] = {}
#-- init start time
try:
StartAt = AIParseUtils.readInt(WLUnit['otherinfo']['StartAt'], DefaultValue = 0)
StartAt = StartAt * 1000 # the time is given is seconds
except KeyError:
StartAt = 0
IBIS_LastRunTime = StartAt
WLUnitMultiplicity = int(WLUnit['multiplicity'])
if bGenerateRandom > 0:
#--- generate random
OneIBISSelectionMethod = IBISSelectionMethod( AppsWithWeightsDic )
OneIBISSelectionMethod.setSelectionMethodByName( SelectionMethodName )
JobIndex = 0
if WLUnitMultiplicity > 1:
NJobs = NJobs * WLUnitMultiplicity
while JobIndex < NJobs:
# 1. select application type
## Application = AIRandomUtils.getRandomWeightedListElement(
## AppsWithWeightsDic,
## ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
## )
Application = OneIBISSelectionMethod.getNextValue()
if Application.find('*') >= 0:
print "IBIS generator: Expanded", Application, "to",
Application = ExpandApplication(Application)
print Application
# 2. select the number of CPU for this job
#TODO: repeat NCPUs selection until App can run on that many CPUs
NCPUs = AIRandomUtils.getRandomWeightedListElement(
TotalCPUsWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
# 3. select the number of components and try to create subcomponents,
# until successfully matching restrictions
WhileCondition = 1
NTries = 100
while WhileCondition != 0 and NTries > 0:
NTries = NTries - 1
NComponents = AIRandomUtils.getRandomWeightedListElement(
NComponentsWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
if NComponents > NCPUs:
continue # impossible match, just retry
NCPUsPerComponent = NCPUs / NComponents
# ensure restrictions are met
if NCPUsPerComponent < MinComponentSize or NCPUsPerComponent > MaxComponentSize:
continue
NRemainingComponents = NCPUs - NCPUsPerComponent * NComponents
if NRemainingComponents > 0 and EqualCPUsPerComponent == 1:
continue
# >= -> including the extra CPU for some components
if NRemainingComponents > 0 and NCPUsPerComponent == MaxComponentSize:
continue
SiteType = AIRandomUtils.getRandomWeightedListElement(
SiteTypesWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
PreComponentsList = []
for ComponentID in xrange(NComponents):
# cpu count
ToAssignHere = NCPUsPerComponent
if ComponentID < NRemainingComponents: # assign the extra CPUs, round-robin
ToAssignHere = ToAssignHere + 1
# site location
if SiteType == 'ordered':
Site = AIRandomUtils.getRandomWeightedListElement(
SitesWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
else:
Site = '*'
# create pre-component information
PreComponent = {}
PreComponent['id'] = ComponentID
PreComponent['count'] = ToAssignHere
PreComponent['location'] = Site
# add pre-component
PreComponentsList.append(PreComponent)
WhileCondition = 0
if WhileCondition == 1: # failed to generate in 100 tries
continue # try again for the same job
#--- generate rnd job
#-- generate rnd job: ID
CurrentUnitID = "%s-%d" % (UnitID, JobIndex % WLUnitMultiplicity)
#-- generate rnd job: index
UnitsDic['info'][JobIndex] = \
generateJobInfo( UnitDir, CurrentUnitID, JobIndex, Application, \
WLUnit, SubmitDurationMS, Submitter )
#-- generate rnd job: components
UnitsDic['jobs'][JobIndex] = \
generateJobComponents(
UnitDir, CurrentUnitID, JobIndex, WLUnit,
PreComponentsList, JavaHomeDir, IbisAppsDir, IbisLibDir,
JavaMaxMemoryLimitMB, Application, NCPUs
)
#-- be more descriptive
UnitsDic['info'][JobIndex]['description'] = \
UnitsDic['info'][JobIndex]['description'] + \
" Components: %d. NCPUs: %d." % (len(PreComponentsList), NCPUs)
#--- done generating
if UnitsDic['jobs'][JobIndex] == None:
NJobs = NJobs - 1 #-- don't ack job, but make sure total no of jobs is decreased
else:
print "IBIS generator: Generated a job of type", Application,
print "with", len(PreComponentsList), "components/",
print NCPUs, "CPUs, in", UnitsDic['info'][JobIndex]['jdf']
JobIndex = JobIndex + 1 #-- ack job
WLUnit['generatedjobs'] = JobIndex
else:
print "Complete workload"
#--- generate all
JobIndex = 0
RepetitionIndex = 0
while RepetitionIndex < WLUnitMultiplicity:
for ComponentSize in ComponentSizeList:
for NumComponents in NumComponentsList:
CurrentUnitID = "%s-%d" % (UnitID, JobIndex % WLUnitMultiplicity)
UnitsDic['info'][JobIndex] = \
generateJobInfo( UnitDir, CurrentUnitID, JobIndex, \
WLUnit, SubmitDurationMS, Submitter )
UnitsDic['jobs'][JobIndex] = \
generateJobComponents( UnitDir, CurrentUnitID, JobIndex, WLUnit, \
NumComponents, ComponentSize, Application, NCPUs )
JobIndex = JobIndex + 1
RepetitionIndex = RepetitionIndex + 1
if JobIndex > NJobs and NJobs > 0:
print "Generated %d jobs and stopped because of the NJobs limit."
break
WLUnit['generatedjobs'] = JobIndex
#raise Exception, "IBIS generator: Not implemented yet"
return UnitsDic
def generateWorkload(UnitDir, UnitID, WLUnit, SubmitDurationMS, bGenerateRandom=1):
global HSH_LastRunTime
# print WLUnit['otherinfo']
# {'RandomWorkload': 'true', 'NComponents': '1-8', 'NJobs': '50', 'ComponentSize':'4-8'}
if bGenerateRandom > 0:
print "Random workload"
else:
print "Full workload"
if "otherinfo" in WLUnit.keys():
try:
NComponents = WLUnit["otherinfo"]["NComponents"]
except KeyError:
raise Exception, "Hashim's generator: No NComponents info specified in the OtherInfo field."
try:
NumComponentsList = AIParseUtils.readIntRange(NComponents)
if len(NumComponentsList) == 0:
NumComponentsList = AIParseUtils.readIntList(NComponents)
if len(NumComponentsList) == 0:
raise Exception, ""
except Exception, e:
raise Exception, "Hashim's generator: Incorrect NComponents format!\n\tExpected NComponents=X-Y, with X,Y integers and X<Y or\n\tNComponents=V1;...;Vn, Vi integer. Got %s." % NComponents
# print ">>>DEBUG>>> NumComponentsList", NumComponentsList
try:
NJobs = WLUnit["otherinfo"]["NJobs"]
except KeyError:
NJobs = "0"
if bGenerateRandom > 0: # -- NJobs required if random, optional otherwise
raise Exception, "Hashim's generator: No NJobs info specified in the OtherInfo field."
try:
NJobs = int(NJobs.strip())
except:
raise Exception, "Hashim's generator: Incorrect NJobs format! Expected NJobs=X, with X integer."
# print ">>>DEBUG>>> NJobs", NJobs
try:
ComponentSize = WLUnit["otherinfo"]["ComponentSize"]
except KeyError:
raise Exception, "Hashim's generator: No ComponentSize info specified in the OtherInfo field."
try:
ComponentSizeList = AIParseUtils.readIntRange(ComponentSize)
if len(ComponentSizeList) == 0:
ComponentSizeList = AIParseUtils.readIntList(ComponentSize)
if len(ComponentSizeList) == 0:
raise Exception, ""
except:
raise Exception, "Hashim's generator: Incorrect ComponentSize format!\n\tExpected ComponentSize=X-Y, with X,Y integers and X<Y or\n\tComponentSize=V1;...;Vn, Vi integer. Got %s." % ComponentSize
# print ">>>DEBUG>>> ComponentSizeList", ComponentSizeList
try:
Submitter = WLUnit["otherinfo"]["Submitter"]
except KeyError:
Submitter = "drunner"
print "WARNING! Hashim's generator: No Submitter info specified in the OtherInfo field.\n\tSetting to default", Submitter
try:
Application = WLUnit["otherinfo"]["Application"]
except KeyError:
Application = "Poisson-g2-gm"
print "WARNING! Hashim's generator: No Application info specified in the OtherInfo field.\n\tSetting to default", Application
try:
Application = WLUnit["otherinfo"]["Application"]
except KeyError:
Application = "Poisson-g2-gm"
print "WARNING! Hashim's generator: No Application info specified in the OtherInfo field.\n\tSetting to default", Application
else:
raise Exception, "Hashim's generator: No OtherInfo data! Expected at least ComponentSize,NJobs,NComponents fields."
UnitsDic = {}
UnitsDic["info"] = {}
UnitsDic["jobs"] = {}
# -- init start time
try:
StartAt = AIParseUtils.readInt(WLUnit["otherinfo"]["StartAt"], DefaultValue=0)
StartAt = StartAt * 1000 # the time is given is seconds
except KeyError:
StartAt = 0
HSH_LastRunTime = StartAt
WLUnitMultiplicity = int(WLUnit["multiplicity"])
if bGenerateRandom > 0:
# --- generate random
JobIndex = 0
if WLUnitMultiplicity > 1:
NJobs = NJobs * WLUnitMultiplicity
while JobIndex < NJobs:
ComponentSize = AIRandomUtils.getRandomListElement(ComponentSizeList)
NumComponents = AIRandomUtils.getRandomListElement(NumComponentsList)
