def generateWorkloadUnit( UnitDir, UnitID, WLUnit, SubmitDurationMS, bGenerateRandom = 1 ):
"""
Out:
UnitsDic
o A dictionary with keys 'info' and 'jobs'.
Notes:
o UnitsDic['info'] contains a dictionary of jobs info,
indexed with an integer counter
o Each job info contains at least the keys
name, description, jdf, submitCommand, runTime
o UnitsDic['jobs'] contains a dictionary of jobs data,
indexed with an integer counter
o Each job data contains the list of components of that job
o Each component in the list is a dictionary with at least the
following fields: executable, stdout, stderr, name, description,
directory, maxWallTime, arguments, env, stagein, stageout
See also:
utils/WLDocHandlers.WLSubmitJobKeys
this_file.SMPI1Component.generateComponent
"""
global SMPI1_LastRunTime
SMPI1Component.SMPI1_Exe = os.path.join( "/tmp", "smpi1t" )
SMPI1Component.SMPI1_ParKSizes = [ "0", "32", "128", "512", "1024" ]
SMPI1Component.SMPI1_ParKSuperSizes = [ "0", "16", "32", "64", "128", "256" ]
SMPI1Component.SMPI1_ParSupersteps = [ "1", "2", "5", "10", "20", "50", "100" ]
SMPI1Component.SMPI1_ParMemoryKItems = [ "10", "25", "50", "100", "250", "500", "1000", "5000" ]
SMPI1Component.SMPI1_ParMemoryElementsPerItem = [ "3", "4", "10", "100", "500", "1000" ]
SMPI1Component.SMPI1_ParXChangeElementsPerStep = [ "100", "500", "1000", "10000", "50000" ]
SMPI1Component.SMPI1_ParComputationPerMemoryItem = [ "2", "10", "100", "1000" ]
SMPI1Component.SMPI1_RunTimeInMinutes = [ "2", "5", "10", "15" ]
if 'otherinfo' in WLUnit.keys():
try:
OneString = WLUnit['otherinfo']['Exe']
SMPI1Component.SMPI1_Exe = OneString
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParKSizes'],
ItemSeparator = ',' )
SMPI1Component.SMPI1_ParKSizes = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParKSuperSizes'],
ItemSeparator = ',' )
SMPI1Component.SMPI1_ParKSuperSizes = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParSupersteps'],
ItemSeparator = ',' )
SMPI1Component.SMPI1_ParSupersteps = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParMemoryKItems'],
ItemSeparator = ',' )
SMPI1Component.SMPI1_ParMemoryKItems = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParMemoryElementsPerItem'],
ItemSeparator = ',' )
SMPI1Component.SMPI1_ParMemoryElementsPerItem = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParXChangeElementsPerStep'],
ItemSeparator = ',' )
SMPI1Component.SMPI1_ParXChangeElementsPerStep = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParComputationPerMemoryItem'],
ItemSeparator = ',' )
SMPI1Component.SMPI1_ParComputationPerMemoryItem = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['RunTimeInMinutes'],
ItemSeparator = ',' )
SMPI1Component.SMPI1_RunTimeInMinutes = OneIntList
except KeyError:
pass
try:
NComponentsWithWeightsDic = \
AIParseUtils.readIntWithWeightsList(
WLUnit['otherinfo']['NComponentsWithWeights'],
DefaultWeight = 1.0
)
# NComponentsWithWeightsDic['Values']; NComponentsWithWeightsDic['TotalWeight']
except KeyError:
raise Exception, "SMPI1 generator: Invalid NComponentsWithWeights info specified in the OtherInfo field."
##print ">>>>>> NComponentsWithWeightsDic", NComponentsWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
try:
TotalCPUsWithWeightsDic = \
AIParseUtils.readIntWithWeightsList(
WLUnit['otherinfo']['TotalCPUsWithWeights'],
DefaultWeight = 1.0
)
except KeyError:
raise Exception, "SMPI1 generator: Invalid TotalCPUsWithWeights info specified in the OtherInfo field."
##print ">>>>>> TotalCPUsWithWeightsDic", TotalCPUsWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
try:
MinComponentSize = AIParseUtils.readInt(WLUnit['otherinfo']['MinComponentSize'], DefaultValue = 0)
except KeyError:
MinComponentSize = 1
#raise Exception, "SMPI1 generator: No MinComponentSize info specified in the OtherInfo field."
try:
MaxComponentSize = AIParseUtils.readInt(WLUnit['otherinfo']['MaxComponentSize'], DefaultValue = 0)
except KeyError:
MaxComponentSize = 10000000
#raise Exception, "SMPI1 generator: No MaxComponentSize info specified in the OtherInfo field."
try:
EqualCPUsPerComponent = AIParseUtils.readBoolean(WLUnit['otherinfo']['SMPI1AppsDir'])
except KeyError:
EqualCPUsPerComponent = 0
#raise Exception, "SMPI1 generator: No EqualCPUsPerComponent info specified in the OtherInfo field. Setting to default " + AIParseUtils.IntToBooleanDic[EqualCPUsPerComponent] + '.'
try:
AppBaseDir = WLUnit['otherinfo']['AppBaseDir']
except KeyError:
AppBaseDir = '/tmp'
#raise Exception, "SMPI1 generator: No AppBaseDir info specified in the OtherInfo field."
print "WARNING! SMPI1 generator: No AppBaseDir info specified in the OtherInfo field.", "Assuming default:", AppBaseDir
try:
Application = WLUnit['otherinfo']['Application']
except KeyError:
Application = 'smpi1t-gm'
#raise Exception, "SMPI1 generator: No AppBaseDir info specified in the OtherInfo field."
print "WARNING! SMPI1 generator: No Application info specified in the OtherInfo field.", "Assuming default:", Application
try:
Submitter = WLUnit['otherinfo']['Submitter']
except KeyError:
Submitter = 'krunner -l DEBUG -g -e -o -f ${JDF}'
print "WARNING! SMPI1 generator: No Submitter info specified in the OtherInfo field.", "Assuming default", Submitter
try:
SiteTypesWithWeightsDic = \
AIParseUtils.readStringWithWeightsList(
WLUnit['otherinfo']['SiteTypesWithWeights'],
DefaultWeight = 1.0
)
except KeyError:
SiteTypesWithWeights = {AIParseUtils.VALUES:[], AIParseUtils.TOTAL_WEIGHT:0.0 }
print "WARNING! SMPI1 generator: No SiteTypesWithWeights info specified in the OtherInfo field.\n\tSetting to default", SitesList
##print ">>>>>> SiteTypesWithWeightsDic", SiteTypesWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
try:
SitesWithWeightsDic = \
AIParseUtils.readStringWithWeightsList(
WLUnit['otherinfo']['SitesWithWeights'],
DefaultWeight = 1.0
)
except KeyError:
SitesWithWeightsDic = {AIParseUtils.VALUES:[], AIParseUtils.TOTAL_WEIGHT:0.0 }
print "WARNING! SMPI1 generator: No SitesWithWeights info specified in the OtherInfo field.\n\tSetting to default", SitesList
##print ">>>>>> SitesWithWeightsDic", SitesWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
else:
raise Exception, "SMPI1 generator: No OtherInfo data! Expected at least ComponentSize,NJobs,NComponents fields."
UnitsDic = {}
UnitsDic['info'] = {}
UnitsDic['jobs'] = {}
#-- init start time
try:
StartAt = AIParseUtils.readInt(WLUnit['otherinfo']['StartAt'], DefaultValue = 0)
StartAt = StartAt * 1000 # the time is given is seconds
except KeyError:
StartAt = 0
SMPI1_LastRunTime = StartAt
if 'FirstJobIndex' in WLUnit:
JobIndex = int(WLUnit['FirstJobIndex'])
else:
JobIndex = 0
WLUnitMultiplicity = int(WLUnit['multiplicity'])
if bGenerateRandom > 0 :
index = 0
while index < WLUnitMultiplicity:
# 1. generate a random site type (unordered or ordered)
SiteType = AIRandomUtils.getRandomWeightedListElement(
SiteTypesWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
# 2. select the number of CPU for this job
#TODO: repeat NCPUs selection until App can run on that many CPUs
NCPUs = AIRandomUtils.getRandomWeightedListElement(
TotalCPUsWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
# 3. select the number of components and try to create subcomponents,
# until successfully matching restrictions
WhileCondition = 1
NTries = 100
while WhileCondition != 0 and NTries > 0:
NTries = NTries - 1
NComponents = AIRandomUtils.getRandomWeightedListElement(
NComponentsWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
if NComponents > NCPUs:
continue # impossible match, just retry
NCPUsPerComponent = NCPUs / NComponents
# ensure restrictions are met
if NCPUsPerComponent < MinComponentSize or NCPUsPerComponent > MaxComponentSize:
continue
NRemainingComponents = NCPUs - NCPUsPerComponent * NComponents
if NRemainingComponents > 0 and EqualCPUsPerComponent == 1:
continue
# >= -> including the extra CPU for some components
if NRemainingComponents > 0 and NCPUsPerComponent == MaxComponentSize:
continue
PreComponentsList = []
for ComponentID in xrange(NComponents):
# cpu count
ToAssignHere = NCPUsPerComponent
if ComponentID < NRemainingComponents: # assign the extra CPUs, round-robin
ToAssignHere = ToAssignHere + 1
# site location
if SiteType == 'ordered':
Site = AIRandomUtils.getRandomWeightedListElement(
SitesWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
else:
Site = '*'
# create pre-component information
PreComponent = {}
PreComponent['id'] = ComponentID
PreComponent['count'] = ToAssignHere
PreComponent['location'] = Site
# add pre-component
PreComponentsList.append(PreComponent)
WhileCondition = 0
if WhileCondition == 1: # failed to generate in 100 tries
continue # try again for the same job
#--- generate rnd job
#-- generate job info
UnitsDic['info'][index] = \
generateJobInfo( UnitDir, UnitID, JobIndex, \
WLUnit, SubmitDurationMS, Submitter )
#-- generate job components
UnitsDic['jobs'][index] = \
generateJobComponents( UnitDir, UnitID, JobIndex, WLUnit, PreComponentsList, \
Application, AppBaseDir, \
bGenerateRandom )
index = index + 1
JobIndex = JobIndex + 1
WLUnit['generatedjobs'] = index
else:
index = 0
for Size in SMPI1Component.SMPI1_ParKSuperSizes:
for N in SMPI1Component.SMPI1_ParSupersteps:
for M in SMPI1Component.SMPI1_ParMemoryKItems:
for S in SMPI1Component.SMPI1_ParMemoryElementsPerItem:
for X in SMPI1Component.SMPI1_ParXChangeElementsPerStep:
for C in SMPI1Component.SMPI1_ParComputationPerMemoryItem:
raise Exception, "ERROR! SMPI1 is not finished!"
#--- generate wall time once for all components of this job
MaxWallTime = AIRandomUtils.getRandomListElement( SMPI1Component.SMPI1_RunTimeInSeconds )
#-- generate a random site type (unordered, ordered, ...)
SiteType = AIRandomUtils.getRandomWeightedListElement(
SiteTypesWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
#-- generate job info
UnitsDic['info'][index] = \
generateJobInfo( UnitDir, UnitID, JobIndex, \
WLUnit, SubmitDurationMS )
#-- generate job components
UnitsDic['jobs'][index] = \
generateJobComponents( UnitDir, UnitID, JobIndex, WLUnit, \
bGenerateRandom, SiteType, Size, \
N, M, S, X, C, MaxWallTime )
index = index + 1
JobIndex = JobIndex + 1
#-- set generated jobs
WLUnit['generatedjobs'] = index
return UnitsDic
def generateWorkloadUnit( UnitDir, UnitID, WLUnit, SubmitDurationMS, bGenerateRandom = 1 ):
"""
Out:
UnitsDic
o A dictionary with keys 'info' and 'jobs'.
Notes:
o UnitsDic['info'] contains a dictionary of jobs info,
indexed with an integer counter
o Each job info contains at least the keys
name, description, jdf, submitCommand, runTime
o UnitsDic['jobs'] contains a dictionary of jobs data,
indexed with an integer counter
o Each job data contains the list of components of that job
o Each component in the list is a dictionary with at least the
following fields: executable, stdout, stderr, name, description,
directory, maxWallTime, arguments, env, stagein, stageout
See also:
utils/WLDocHandlers.WLSubmitJobKeys
this_file.SSERComponent.generateComponent
"""
global SSER_LastRunTime
SSERComponent.SSER_Exe = "/home/antoniou/Synthetic_Apps/sseriot" #os.path.join( "/tmp", "ssert" )
SSERComponent.SSER_ParKSizes = [ "0", "32", "128", "512", "1024" ]
SSERComponent.SSER_ParKSuperSizes = [ "0", "16", "32", "64", "128", "256" ]
SSERComponent.SSER_ParSupersteps = [ "1", "2", "5", "10", "20", "50", "100" ]
SSERComponent.SSER_ParMemoryKItems = [ "10", "25", "50", "100", "250", "500", "1000", "5000" ]
SSERComponent.SSER_ParMemoryElementsPerItem = [ "3", "4", "10", "100", "500", "1000" ]
SSERComponent.SSER_ParComputationPerMemoryItem = [ "2", "10", "100", "1000" ]
SSERComponent.SSER_RunTimeInMinutes = [ "2", "5", "10", "15" ]
if 'otherinfo' in WLUnit.keys():
try:
OneString = WLUnit['otherinfo']['Exe']
SSERComponent.SSER_Exe = OneString
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParKSizes'],
ItemSeparator = ',' )
SSERComponent.SSER_ParKSizes = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParKSuperSizes'],
ItemSeparator = ',' )
SSERComponent.SSER_ParKSuperSizes = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParSupersteps'],
ItemSeparator = ',' )
SSERComponent.SSER_ParSupersteps = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParMemoryKItems'],
ItemSeparator = ',' )
SSERComponent.SSER_ParMemoryKItems = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParMemoryElementsPerItem'],
ItemSeparator = ',' )
SSERComponent.SSER_ParMemoryElementsPerItem = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParComputationPerMemoryItem'],
ItemSeparator = ',' )
SSERComponent.SSER_ParComputationPerMemoryItem = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['RunTimeInMinutes'],
ItemSeparator = ',' )
SSERComponent.SSER_RunTimeInMinutes = OneIntList
except KeyError:
pass
UnitsDic = {}
UnitsDic['info'] = {}
UnitsDic['jobs'] = {}
#-- init start time
try:
StartAt = AIParseUtils.readInt(WLUnit['otherinfo']['StartAt'], DefaultValue = 0)
StartAt = StartAt * 1000 # the time is given is seconds
except KeyError:
StartAt = 0
SSER_LastRunTime = StartAt
if 'FirstJobIndex' in WLUnit:
JobIndex = int(WLUnit['FirstJobIndex'])
else:
JobIndex = 0
if bGenerateRandom > 0 :
index = 0
maxIndex = WLUnit['multiplicity']
while index < maxIndex:
UnitsDic['info'][index] = \
generateJobInfo( UnitDir, UnitID, JobIndex, \
WLUnit, SubmitDurationMS )
UnitsDic['jobs'][index] = \
generateJobComponents( UnitDir, UnitID, JobIndex, WLUnit, \
bGenerateRandom )
index = index + 1
JobIndex = JobIndex + 1
WLUnit['generatedjobs'] = index
else:
index = 0
for Size in SSERComponent.SSER_ParKSuperSizes:
for N in SSERComponent.SSER_ParSupersteps:
for M in SSERComponent.SSER_ParMemoryKItems:
for S in SSERComponent.SSER_ParMemoryElementsPerItem:
for C in SSERComponent.SSER_ParComputationPerMemoryItem:
for MaxWallTime in SSERComponent.SSER_RunTimeInSeconds:
UnitsDic['info'][index] = \
generateJobInfo( UnitDir, UnitID, JobIndex, \
WLUnit, SubmitDurationMS )
UnitsDic['jobs'][index] = \
generateJobComponents( UnitDir, UnitID, JobIndex, WLUnit, \
bGenerateRandom, Size, N, M, S, C, MaxWallTime)
index = index + 1
JobIndex = JobIndex + 1
#-- set generated jobs
WLUnit['generatedjobs'] = index
return UnitsDic
def generateWorkload(UnitDir, UnitID, WLUnit, SubmitDurationMS, bGenerateRandom=1):
global HSH_LastRunTime
# print WLUnit['otherinfo']
# {'RandomWorkload': 'true', 'NComponents': '1-8', 'NJobs': '50', 'ComponentSize':'4-8'}
if bGenerateRandom > 0:
print "Random workload"
else:
print "Full workload"
if "otherinfo" in WLUnit.keys():
try:
NComponents = WLUnit["otherinfo"]["NComponents"]
except KeyError:
raise Exception, "Hashim's generator: No NComponents info specified in the OtherInfo field."
try:
NumComponentsList = AIParseUtils.readIntRange(NComponents)
if len(NumComponentsList) == 0:
NumComponentsList = AIParseUtils.readIntList(NComponents)
if len(NumComponentsList) == 0:
raise Exception, ""
except Exception, e:
raise Exception, "Hashim's generator: Incorrect NComponents format!\n\tExpected NComponents=X-Y, with X,Y integers and X<Y or\n\tNComponents=V1;...;Vn, Vi integer. Got %s." % NComponents
# print ">>>DEBUG>>> NumComponentsList", NumComponentsList
try:
NJobs = WLUnit["otherinfo"]["NJobs"]
except KeyError:
NJobs = "0"
if bGenerateRandom > 0: # -- NJobs required if random, optional otherwise
raise Exception, "Hashim's generator: No NJobs info specified in the OtherInfo field."
try:
NJobs = int(NJobs.strip())
except:
raise Exception, "Hashim's generator: Incorrect NJobs format! Expected NJobs=X, with X integer."
# print ">>>DEBUG>>> NJobs", NJobs
try:
ComponentSize = WLUnit["otherinfo"]["ComponentSize"]
except KeyError:
raise Exception, "Hashim's generator: No ComponentSize info specified in the OtherInfo field."
try:
ComponentSizeList = AIParseUtils.readIntRange(ComponentSize)
if len(ComponentSizeList) == 0:
ComponentSizeList = AIParseUtils.readIntList(ComponentSize)
if len(ComponentSizeList) == 0:
raise Exception, ""
except:
raise Exception, "Hashim's generator: Incorrect ComponentSize format!\n\tExpected ComponentSize=X-Y, with X,Y integers and X<Y or\n\tComponentSize=V1;...;Vn, Vi integer. Got %s." % ComponentSize
# print ">>>DEBUG>>> ComponentSizeList", ComponentSizeList
try:
Submitter = WLUnit["otherinfo"]["Submitter"]
except KeyError:
Submitter = "drunner"
print "WARNING! Hashim's generator: No Submitter info specified in the OtherInfo field.\n\tSetting to default", Submitter
try:
Application = WLUnit["otherinfo"]["Application"]
except KeyError:
Application = "Poisson-g2-gm"
print "WARNING! Hashim's generator: No Application info specified in the OtherInfo field.\n\tSetting to default", Application
