def generateJobComponents(UnitDir, UnitID, JobIndex, WLUnit, NumComponents, ComponentSize, Application):
# minutes: 3, 15, 30, 700
RunTimeInMinutesList = ["3", "15", "30", "700"]
ComponentsList = []
# --- generate wall time once for all components of this job
MaxWallTime = AIRandomUtils.getRandomListElement(RunTimeInMinutesList)
if UnitDir[0] != "/":
if sys.platform.find("linux") >= 0:
UnitDir = os.path.join(os.environ["PWD"], UnitDir)
else:
UnitDir = os.path.join(os.getcwd(), UnitDir)
# -- hashim's workload definition
ComponentDirName = "%d-%d-%d" % (JobIndex, NumComponents, ComponentSize)
FullComponentDirName = os.path.join(UnitDir, "%s/" % UnitID, ComponentDirName)
# --- Create output directory, if it does not exist
if os.path.exists(FullComponentDirName):
if not os.path.isdir(FullComponentDirName):
print "Output for job", JobIndex, "(" + FullComponentDirName + ")", "exists, but is not a directory", "...skipping job"
return -1
else:
try:
os.makedirs(FullComponentDirName)
except OSError, e:
print "Cannot create output directory for job", JobIndex, "...skipping job"
print "\tOS returned:", e
return -1
def generateWorkload( OutDir, FullWLFileName, ListOfDefs, SubmitDurationMS,
AUnitGensLoader, AWLGensLoader, AJDFGeneratorsLoader, JDFGeneratorNames,
DictionaryOfSites ,arrivalDistribution = None):
print "STATUS! Starting workload generation!"
# the number of workflow engines for which we generate output files
NEngines = len(JDFGeneratorNames)
print "STATUS! Generating %d workload formats \n" % NEngines
FileFormatsArray = []
#DagCommandsFormatsMap = {'condor_dag_wrapper.sh':'dagman', 'karajan_wrapper_remote.sh':'karajan'}
#--- reset random generator
AIRandomUtils.initRandom()
#--- init arrival time generator
try:
ArrivalTimeRandom = AIRandomUtils.AIRandom( time() )
except Exception, e:
print "ERROR! Cannot instantiate the ArrivalTime random generator.", e
def getNextValue(self):
if self.SelectionMethod == IBISSelectionMethod.SM_Random:
return self.RandomGenerator.getRandomListElement( self.DataList )
elif self.SelectionMethod == IBISSelectionMethod.SM_WeightedRandom:
return AIRandomUtils.getRandomWeightedListElement(
self.DataDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
elif self.SelectionMethod == IBISSelectionMethod.SM_RoundRobin:
if self.Counter >= self.MaxCounter: self.Counter = 0
try:
ReturnValue = self.DataDic[AIParseUtils.VALUES][self.Counter][0]
except IndexError, e:
print '>>>>>>>>>', e
print '>>TRACE>>', traceback.print_exc()
print '>>CNTR>>>', self.Counter, "/", self.MaxCounter
self.Counter = self.Counter + 1
return ReturnValue
def generateJobComponents( UnitDir, UnitID, JobIndex, WLUnit, \
PreComponentsList, JavaHomeDir, IbisAppsDir, IbisLibDir, \
JavaMaxMemoryLimitMB, Application, NCPUs):
# minutes: 3, 15, 30, 700
RunTimeInMinutesList = [ '3', '15', '30', '700' ]
NumComponents = len(PreComponentsList)
ComponentsList = []
#--- generate wall time once for all components of this job
MaxWallTime = AIRandomUtils.getRandomListElement( RunTimeInMinutesList )
if UnitDir[0] != '/':
if sys.platform.find("linux") >= 0:
UnitDir = os.path.join( os.environ['PWD'], UnitDir )
else:
UnitDir = os.path.join( os.getcwd(), UnitDir )
### THE UGLY HACK: convert /disk1/home3/koala5/grenchmark to /home/koala5/grenchmark
pos = UnitDir.find('/koala')
if pos >= 0:
UnitDir = '/home' + UnitDir[pos:]
#-- IBIS workload definition
ComponentDirName = "%d-%d" % ( JobIndex, NumComponents )
FullComponentDirName = os.path.join( UnitDir, "%s/" % UnitID, ComponentDirName )
#--- Create output directory, if it does not exist
if os.path.exists( FullComponentDirName ):
if not os.path.isdir( FullComponentDirName ):
print "Output for job", JobIndex, "("+FullComponentDirName+")", "exists, but is not a directory", "...skipping job"
return None
else:
try:
os.makedirs( FullComponentDirName )
except OSError, e:
print "Cannot create output directory for job", JobIndex, "...skipping job"
print '\tOS returned:', e
return None
def ExpandApplication(Application):
"""
replace an ambiguous application name (*) with a real name
(randomly selected from the global applications list)
"""
#-- find all matching names
FullNamesList = []
#-- replace the user friendly '*' with the Python RegExp correspondent
Application = Application.replace('*', '\w+') # re's synthax requires + for 1/1+ matches
SearchRE = re.compile(Application)
#-- create the list of applications whose name match the request
for App in IBIS_Apps.keys():
if SearchRE.search('^' + App):
FullNamesList.append(App)
#-- select one name or die
#print FullNamesList
if len(FullNamesList) == 0:
raise Exception, 'IBIS generator: Wrong application name ' + Application + ' (cannot expand).'
FullApplicationName = AIRandomUtils.getRandomListElement(FullNamesList)
return FullApplicationName
def generateComponent( self, bGenerateRandom = 1, \
Size = 0, N = 0, M = 0, S = 0, C = 0, MaxWallTime = 0 ):
"""
Generates one component using SSER as the application.
This method does NOT write a physical JDF, but generates
all the needed parameters, directories, and input files
instead. The WLMain.generateWorkload is responsible for
actualy writing the JDFs.
In:
bGenerateRandom -- whether this component is to be generated randomly
(>0 for true, <=0 for false)
size, N, M, S, C, MaxWallTime -- only valid if bGenerateRandom is 0
Return:
Int, >=0 on success, <0 otherwise
Notes:
o Upon success, the ComponentData will contain at least the following
fields:
executable -- the complete path to an executable file
stdout -- a file that will receive the standard output messages
stderr -- a file that will receive the standard error messages
name -- a name for this component (if unique, the better)
description -- a description of this component
directory -- the directory where the component should run
maxWallTime -- the max time requested this app should run
arguments -- the list of arguments to be fed to the component's application
env -- the list of environmental variables, as (Name, Value) tuples
stagein -- a list of files to be staged in
stageout -- a list of files to be staged out
"""
if bGenerateRandom > 0:
if self.HaveInput > 0:
InputSize = AIRandomUtils.getRandomListElement( SSERComponent.SSER_ParKSuperSizes )
else:
InputSize = 0
if self.HaveOutput > 0:
OutputSize = AIRandomUtils.getRandomListElement( SSERComponent.SSER_ParKSuperSizes )
else:
OutputSize = 0
N = AIRandomUtils.getRandomListElement( SSERComponent.SSER_ParSupersteps )
M = AIRandomUtils.getRandomListElement( SSERComponent.SSER_ParMemoryKItems )
S = AIRandomUtils.getRandomListElement( SSERComponent.SSER_ParMemoryElementsPerItem )
C = AIRandomUtils.getRandomListElement( SSERComponent.SSER_ParComputationPerMemoryItem )
# N = SSERComponent.SSER_ParSupersteps[2]
# M = AIRandomUtils.getRandomListElement( SSERComponent.SSER_ParMemoryKItems )
# S = AIRandomUtils.getRandomListElement( SSERComponent.SSER_ParMemoryElementsPerItem )
# C = SSERComponent.SSER_ParComputationPerMemoryItem[3]
MaxWallTime = AIRandomUtils.getRandomListElement( SSERComponent.SSER_RunTimeInMinutes )
if self.UnitDir[0] != '/':
if sys.platform.find("linux") >= 0:
self.UnitDir = os.path.join( os.environ['PWD'], self.UnitDir )
else:
self.UnitDir = os.path.join( os.getcwd(), self.UnitDir )
## too long component dir name
#ComponentDirName = "%s_sser_%dx_%d_i%d_o%d" % \
# (self.ComponentData['id'], self.ComponentData['count'], int(N), int(InputSize), int(OutputSize))
ComponentDirName = "%s_sser" % self.ComponentData['id']
FullComponentDirName = os.path.join( self.UnitDir, ComponentDirName )
#--- Create output directory, if it does not exist
if os.path.exists( FullComponentDirName ):
if not os.path.isdir( FullComponentDirName ):
print "Output for job", self.ComponentData['id'], "("+FullComponentDirName+")", "exists, but is not a directory", "...skipping job"
return -1
else:
try:
os.makedirs( FullComponentDirName )
except OSError, e:
print "Cannot create output directory for job", self.ComponentData['id'] , "...skipping job"
print '\tOS returned:', e
return -1
SSER_LastRunTime = RunTime
InfoDic['runTime'] = "%.3f" % RunTime
print "In sser-unit: RUNTIME IS ",InfoDic['runTime']
except AIRandomUtils.AIRandomError, e:
print "AIRandomUtils.AIRandomError", e
InfoDic['runTime'] = None
except Exception, e:
print "ERROR!", e, "Name", Name, "Params", ParamsList
InfoDic['runTime'] = None
if (InfoDic['runTime'] == None) or \
(float(InfoDic['runTime']) < 0) or (float(InfoDic['runTime']) > SubmitDurationMS):
# all workload units are started within at most SubmitDurationMS miliseconds
print "Wrong arrival time of", InfoDic['runTime'], "generated by", Name, ParamsList
print "Resetting to default U(0,%d)" % SubmitDurationMS
InfoDic['runTime'] = str(AIRandomUtils.getRandomInt(0, SubmitDurationMS))
return InfoDic
def generateJobComponents( UnitDir, UnitID, JobIndex, WLUnit, bGenerateRandom = 1, \
Size = 0, N = 20, M = None, S = None, C = 10, MaxWallTime = 0):
ComponentsList = []
nValidComponents = 0
for ComponentID in WLUnit['components'].keys():
Component = WLUnit['components'][ComponentID]
WLCompName = "%s-%d-%s" % ( UnitID, JobIndex, ComponentID ) # added multiplicity id
#-- set component
ComponentData = {}
if bGenerateRandom > 0:
def generateWorkload( UnitDir, UnitID, WLUnit, SubmitDurationMS, bGenerateRandom = 1 ):
global SWF_LastRunTime
SWFConstsInstance = SWFConsts()
CurrentSWFParams = readParams( WLUnit )
for Filter in CurrentSWFParams.SWF.Filters:
print 'Filter', Filter, '->', CurrentSWFParams.SWF.Filters[Filter]
JobsList, UNIXStartTime = readSWFFile( CurrentSWFParams.SWF.File, CurrentSWFParams )
print "After filtering out,", len(JobsList), "jobs remained."
##for Job in JobsList:
## print Job
UnitsDic = {}
UnitsDic['info'] = {}
UnitsDic['jobs'] = {}
#-- init start time
try:
StartAt = AIParseUtils.readInt(WLUnit['otherinfo']['StartAt'], DefaultValue = 0)
StartAt = StartAt * 1000 # the time is given is seconds
except KeyError:
StartAt = 0
SWF_LastRunTime = StartAt
WLUnitMultiplicity = int(WLUnit['multiplicity'])
FirstJob = None
JobIndex = 0
NJobsInList = len(JobsList)
for GroupID in xrange(WLUnitMultiplicity):
for JobID in xrange(NJobsInList):
#JobIndex = JobID + GroupID * NJobsInList
#--- get trace data
Job = JobsList[JobID]
##print Job
if FirstJob is None:
FirstJob = Job
# 2. select the number of CPU for this job
#TODO: repeat NCPUs selection until App can run on that many CPUs
NCPUs = Job[SWFConstsInstance.NCPUs]
# 3. select the number of components and try to create subcomponents,
# until successfully matching restrictions
WhileCondition = 1
NTries = 250
while WhileCondition != 0 and NTries > 0:
NTries = NTries - 1
NComponents = AIRandomUtils.getRandomWeightedListElement(
CurrentSWFParams.NComponentsWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
if NComponents > NCPUs:
continue # impossible match, just retry
NCPUsPerComponent = NCPUs / NComponents
# ensure restrictions are met
if NCPUsPerComponent < CurrentSWFParams.MinComponentSize or \
NCPUsPerComponent > CurrentSWFParams.MaxComponentSize:
continue
NRemainingComponents = NCPUs - NCPUsPerComponent * NComponents
if NRemainingComponents > 0 and CurrentSWFParams.EqualCPUsPerComponent == 1:
continue
# >= -> including the extra CPU for some components
if NRemainingComponents > 0 and NCPUsPerComponent == CurrentSWFParams.MaxComponentSize:
continue
SiteType = AIRandomUtils.getRandomWeightedListElement(
CurrentSWFParams.SiteTypesWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
PreComponentsList = []
for ComponentID in xrange(NComponents):
# cpu count
ToAssignHere = NCPUsPerComponent
if ComponentID < NRemainingComponents: # assign the extra CPUs, round-robin
ToAssignHere = ToAssignHere + 1
# site location
if SiteType == 'ordered':
Site = AIRandomUtils.getRandomWeightedListElement(
CurrentSWFParams.SitesWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
else:
Site = '*'
# create pre-component information
PreComponent = {}
PreComponent['id'] = ComponentID
PreComponent['count'] = ToAssignHere
PreComponent['location'] = Site
# add pre-component
PreComponentsList.append(PreComponent)
WhileCondition = 0
# skip errors
if WhileCondition != 0:
continue
#--- generate job
Application = CurrentSWFParams.Application
Submitter = CurrentSWFParams.Submitter
ApplicationBinaryFullPath = os.path.join(CurrentSWFParams.AppBaseDir, CurrentSWFParams.Application)
SWFConstsInstace = SWFConsts()
# convert the start time from ms to s
JobStartTime = 1000 * (Job[SWFConstsInstace.SubmitTime] - FirstJob[SWFConstsInstace.SubmitTime])
if JobStartTime < 0:
print "Flurry! JobStartTime < 0"
print "FirstJob=", FirstJob
print "Job=", Job
#-- generate job: ID
CurrentUnitID = "%s-%d-%d" % (UnitID, GroupID, JobIndex % NJobsInList)
#-- generate job: index
UnitsDic['info'][JobIndex] = \
generateJobInfo( UnitDir, CurrentUnitID, JobIndex, Application, \
WLUnit, SubmitDurationMS, Submitter, \
CurrentSWFParams.UseArrivalTimeDistribution, JobStartTime )
#-- generate job: components
UnitsDic['jobs'][JobIndex] = generateJobComponents(
UnitDir, CurrentUnitID, JobIndex, WLUnit,
PreComponentsList, ApplicationBinaryFullPath, \
NCPUs, Job, CurrentSWFParams.MaxWallTime )
#-- be more descriptive
UnitsDic['info'][JobIndex]['description'] = \
UnitsDic['info'][JobIndex]['description'] + \
" Components: %d. NCPUs: %d." % (len(PreComponentsList), NCPUs)
#--- done generating
if UnitsDic['jobs'][JobIndex] == None:
NJobs = NJobs - 1 #-- don't ack job, but make sure total no of jobs is decreased
else:
print "SWF generator: Generated a job of type", Application,
print "with", len(PreComponentsList), "components/",
print NCPUs, "CPUs, in", UnitsDic['info'][JobIndex]['jdf']
JobIndex = JobIndex + 1
# add the delay between separate units
SWF_LastRunTime = SWF_LastRunTime + CurrentSWFParams.DelayBetweenUnits
print "Generated", JobIndex, "jobs."
WLUnit['generatedjobs'] = JobIndex
#raise Exception, "SWF generator: Not implemented yet"
return UnitsDic
def generateJDF(OutFileName, ListOfComponents):
"""
Generate a OGF JSDL File for a given workload unit.
In:
OutFileName -- the target file for writing the JDF of this workload unit
ListOfComponents -- a list of components
Each component is a dictionary and has defined at
least the following keys:
id, executable, count, description, directory,
maxWallTime, arguments, env, stagein, stageout,
stdout, stderr
"""
print "STATUS! Write JDF file", OutFileName,
OutFile = open(OutFileName, "w")
index = 0
nComponents = len(ListOfComponents)
Component = ListOfComponents[0] # one component
writeJsdlHeader(OutFile)
OutFile.write("<jsdl:JobDescription>\n")
"""
<jsdl:Application>
<jsdl:ApplicationName>ls</jsdl:ApplicationName>
<jsdl-posix:POSIXApplication>
<jsdl-posix:Executable>/bin/ls</jsdl-posix:Executable>
<jsdl-posix:Argument>-la file.txt</jsdl-posix:Argument>
<jsdl-posix:Environment name="LD_LIBRARY_PATH">/usr/local/lib</jsdl-posix:Environment>
<jsdl-posix:Input>/dev/null</jsdl-posix:Input>
<jsdl-posix:Output>stdout.${JOB_ID}</jsdl-posix:Output>
<jsdl-posix:Error>stderr.${JOB_ID}</jsdl-posix:Error>
</jsdl-posix:POSIXApplication>
</jsdl:Application>
"""
OutFile.write("<jsdl:Application>\n")
OutFile.write("<jsdl:ApplicationName>")
OutFile.write("sser" + str(AIRandomUtils.getRandomInt(0, int(2 ** 64))))
OutFile.write("</jsdl:ApplicationName>\n")
OutFile.write("<jsdl-posix:POSIXApplication>\n")
OutFile.write("<jsdl-posix:Executable>")
OutFile.write(str(Component["executable"]))
OutFile.write("</jsdl-posix:Executable>\n")
args = Component["arguments"]
writeApplicationArguments(OutFile, args)
stdout = Component["stdout"]
OutFile.write("<jsdl-posix:Output>" + stdout + "</jsdl-posix:Output>\n")
stderr = Component["stderr"]
OutFile.write("<jsdl-posix:Error>" + stderr + "</jsdl-posix:Error>\n")
# OutFile.write('<jsdl-posix:Input></jsdl-posix:Input>\n')
writeApplicationEnvVariables(OutFile, Component["env"])
OutFile.write('<jsdl-posix:Environment name="LD_LIBRARY_PATH">/usr/local/lib</jsdl-posix:Environment>')
OutFile.write("\n</jsdl-posix:POSIXApplication>\n")
OutFile.write("</jsdl:Application>\n")
"""
<jsdl:DataStaging>
<jsdl:FileName>file.txt</jsdl:FileName>
<jsdl:CreationFlag>overwrite</jsdl:CreationFlag>
<jsdl:DeleteOnTermination>true</jsdl:DeleteOnTermination>
<jsdl:Source>
<jsdl:URI>gsiftp://hydrus.dacya.ucm.es/home/jose/file1.txt</jsdl:URI>
</jsdl:Source>
</jsdl:DataStaging>
"""
stagein = Component["stagein"]
stageout = Component["stageout"]
writeStageInFiles(OutFile, stagein)
writeStageOutFiles(OutFile, stageout)
OutFile.write("</jsdl:JobDescription>")
writeJsdlFooter(OutFile)
OutFile.close()
print "...done"
def runJob(data):
# 1. check if the job can be run (deps & time)
# 1.1. yes: run the job
# 1.1.1. if job run successfully
# 1.1.1.1. yes: mark job as correct
# 1.1.1.2. no: put the job back into the pool, if not failed (too many failures)
# 1.2. no: put the job back into the pool
# -- compute due sleep time - modified (corina): sleep time will always be 0
# sleepDuration = max( 0, data['firstSubmission'] + data['startTime'] - time.time())
sleepDuration = 0
# 1. check if the job can be run (time)
if sleepDuration <= 0: # job is due to run
# -- start timers
result = {}
result["start"] = time.time()
result["wakeup"] = time.time()
result["submit"] = time.time()
# 1. check if the job can be run (time)
JobID = data["id"]
# print "##### [runJob] job is due to run: %s " % JobID
CurrentCompositeApplicationData = data[".CompositeApplicationData"]
if CurrentCompositeApplicationData.isRunnable(JobID): # job can run
# 1.1. yes: run the job
patchSleepDuration = AIRandomUtils.getRandomInt(0, 10)
time.sleep(patchSleepDuration)
Command = data["commandLine"]
outputString = "(" + data["id"] + ") runs command:\n " + Command
print outputString
if data["NoSubmit"] != 1:
try:
cmdOutput = AISystemUtil.getCommandOutput2(Command)
result["status"] = "SUCCEEDED"
CurrentCompositeApplicationData.triggerJobFinished(JobID)
except RuntimeError, e:
print "Job", JobID, "returned:", e
cmdOutput = ""
# job failed ?
CurrentCompositeApplicationData.triggerJobFailure(JobID)
if CurrentCompositeApplicationData.isFailed(JobID):
result["status"] = "FAILED"
CurrentCompositeApplicationData.propagateFailure(JobID)
print "Job", JobID, "was declared failed (too many failures)"
else:
result["status"] = "RETRY"
print "Job", JobID, "failed... rescheduling"
return None
outputString = "Output (" + JobID + "): '" + cmdOutput + "'"
print outputString
else: # no submit -> just mark as succeessful
result["status"] = "SUCCEEDED"
result["exit"] = time.time()
# -- return all the times
##print "Job", JobID, "Send good result", result['status']
return result
elif CurrentCompositeApplicationData.isFailed(JobID):
result["exit"] = time.time()
result["status"] = "FAILED"
# -- return all the times
##print "Job", JobID, "Send good result", result['status']
return result
def getWLName( Prefix = "WL" ):
return "%s-%s-%3.3d" % \
( Prefix, strftime('%Y-%m-%d_%H-%M', gmtime(time()) ),
AIRandomUtils.getRandomInt(0,999) )
#--- create subunits, one per composite structure
SubUnit = {}
SubUnit['.CompositeStructureIndex'] = CompositeStructureIndex
SubUnit['.CompositeStructureID'] = CompositeStructure.getID()
SubUnit['.NJobsToGenerate'] = CompositeStructure.getNNodes()
SubUnit['id'] = "%s" % CompositeStructure.getID()
SubUnit['IsWorkload'] = 0
SubUnit['composite'] = 0
SubUnit['multiplicity'] = 1
# NOTE: all workload units need to use this information to generate correctly
# their JDF names...
#SubUnit['FirstJobIndex'] = JobIndex
CurrentAppType = AIRandomUtils.getRandomWeightedListElement(
AppTypesWithWeights,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
SubUnit['apptype'] = CurrentAppType
#SubUnit['arrivaltimeinfo'] = ('Zero', []) # all jobs in the same composite application arrive at the same time, 0
SubUnit['arrivaltimeinfo'] = WLUnit['arrivaltimeinfo']
SubUnit['arrivaltimefunc'] = WLUnit['arrivaltimefunc']
SubUnit['otherinfo'] = {}
for item in WLUnit['otherinfo']:
CurrentAppStamp = CurrentAppType + '.'
if item.find(CurrentAppStamp) == 0: # this other info item pertains the current application type
EntryInternalName = item[len(CurrentAppStamp):]
SubUnit['otherinfo'][EntryInternalName] = WLUnit['otherinfo'][item]
# TODO: composite apps site info quick-hack fix !!!
def generateWorkloadUnit( UnitDir, UnitID, WLUnit, SubmitDurationMS, bGenerateRandom = 1 ):
"""
Out:
UnitsDic
o A dictionary with keys 'info' and 'jobs'.
Notes:
o UnitsDic['info'] contains a dictionary of jobs info,
indexed with an integer counter
o Each job info contains at least the keys
name, description, jdf, submitCommand, runTime
o UnitsDic['jobs'] contains a dictionary of jobs data,
indexed with an integer counter
o Each job data contains the list of components of that job
o Each component in the list is a dictionary with at least the
following fields: executable, stdout, stderr, name, description,
directory, maxWallTime, arguments, env, stagein, stageout
See also:
utils/WLDocHandlers.WLSubmitJobKeys
this_file.SMPI1Component.generateComponent
"""
global SMPI1_LastRunTime
SMPI1Component.SMPI1_Exe = os.path.join( "/tmp", "smpi1t" )
SMPI1Component.SMPI1_ParKSizes = [ "0", "32", "128", "512", "1024" ]
SMPI1Component.SMPI1_ParKSuperSizes = [ "0", "16", "32", "64", "128", "256" ]
SMPI1Component.SMPI1_ParSupersteps = [ "1", "2", "5", "10", "20", "50", "100" ]
SMPI1Component.SMPI1_ParMemoryKItems = [ "10", "25", "50", "100", "250", "500", "1000", "5000" ]
SMPI1Component.SMPI1_ParMemoryElementsPerItem = [ "3", "4", "10", "100", "500", "1000" ]
SMPI1Component.SMPI1_ParXChangeElementsPerStep = [ "100", "500", "1000", "10000", "50000" ]
SMPI1Component.SMPI1_ParComputationPerMemoryItem = [ "2", "10", "100", "1000" ]
SMPI1Component.SMPI1_RunTimeInMinutes = [ "2", "5", "10", "15" ]
if 'otherinfo' in WLUnit.keys():
try:
OneString = WLUnit['otherinfo']['Exe']
SMPI1Component.SMPI1_Exe = OneString
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParKSizes'],
ItemSeparator = ',' )
SMPI1Component.SMPI1_ParKSizes = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParKSuperSizes'],
ItemSeparator = ',' )
SMPI1Component.SMPI1_ParKSuperSizes = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParSupersteps'],
ItemSeparator = ',' )
SMPI1Component.SMPI1_ParSupersteps = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParMemoryKItems'],
ItemSeparator = ',' )
SMPI1Component.SMPI1_ParMemoryKItems = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParMemoryElementsPerItem'],
ItemSeparator = ',' )
SMPI1Component.SMPI1_ParMemoryElementsPerItem = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParXChangeElementsPerStep'],
ItemSeparator = ',' )
SMPI1Component.SMPI1_ParXChangeElementsPerStep = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['ParComputationPerMemoryItem'],
ItemSeparator = ',' )
SMPI1Component.SMPI1_ParComputationPerMemoryItem = OneIntList
except KeyError:
pass
try:
OneIntList = AIParseUtils.readIntList(
Text = WLUnit['otherinfo']['RunTimeInMinutes'],
ItemSeparator = ',' )
SMPI1Component.SMPI1_RunTimeInMinutes = OneIntList
except KeyError:
pass
try:
NComponentsWithWeightsDic = \
AIParseUtils.readIntWithWeightsList(
WLUnit['otherinfo']['NComponentsWithWeights'],
DefaultWeight = 1.0
)
# NComponentsWithWeightsDic['Values']; NComponentsWithWeightsDic['TotalWeight']
except KeyError:
raise Exception, "SMPI1 generator: Invalid NComponentsWithWeights info specified in the OtherInfo field."
##print ">>>>>> NComponentsWithWeightsDic", NComponentsWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
try:
TotalCPUsWithWeightsDic = \
AIParseUtils.readIntWithWeightsList(
WLUnit['otherinfo']['TotalCPUsWithWeights'],
DefaultWeight = 1.0
)
except KeyError:
raise Exception, "SMPI1 generator: Invalid TotalCPUsWithWeights info specified in the OtherInfo field."
##print ">>>>>> TotalCPUsWithWeightsDic", TotalCPUsWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
try:
MinComponentSize = AIParseUtils.readInt(WLUnit['otherinfo']['MinComponentSize'], DefaultValue = 0)
except KeyError:
MinComponentSize = 1
#raise Exception, "SMPI1 generator: No MinComponentSize info specified in the OtherInfo field."
try:
MaxComponentSize = AIParseUtils.readInt(WLUnit['otherinfo']['MaxComponentSize'], DefaultValue = 0)
except KeyError:
MaxComponentSize = 10000000
#raise Exception, "SMPI1 generator: No MaxComponentSize info specified in the OtherInfo field."
try:
EqualCPUsPerComponent = AIParseUtils.readBoolean(WLUnit['otherinfo']['SMPI1AppsDir'])
except KeyError:
EqualCPUsPerComponent = 0
#raise Exception, "SMPI1 generator: No EqualCPUsPerComponent info specified in the OtherInfo field. Setting to default " + AIParseUtils.IntToBooleanDic[EqualCPUsPerComponent] + '.'
try:
AppBaseDir = WLUnit['otherinfo']['AppBaseDir']
except KeyError:
AppBaseDir = '/tmp'
#raise Exception, "SMPI1 generator: No AppBaseDir info specified in the OtherInfo field."
print "WARNING! SMPI1 generator: No AppBaseDir info specified in the OtherInfo field.", "Assuming default:", AppBaseDir
try:
Application = WLUnit['otherinfo']['Application']
except KeyError:
Application = 'smpi1t-gm'
#raise Exception, "SMPI1 generator: No AppBaseDir info specified in the OtherInfo field."
print "WARNING! SMPI1 generator: No Application info specified in the OtherInfo field.", "Assuming default:", Application
try:
Submitter = WLUnit['otherinfo']['Submitter']
except KeyError:
Submitter = 'krunner -l DEBUG -g -e -o -f ${JDF}'
print "WARNING! SMPI1 generator: No Submitter info specified in the OtherInfo field.", "Assuming default", Submitter
try:
SiteTypesWithWeightsDic = \
AIParseUtils.readStringWithWeightsList(
WLUnit['otherinfo']['SiteTypesWithWeights'],
DefaultWeight = 1.0
)
except KeyError:
SiteTypesWithWeights = {AIParseUtils.VALUES:[], AIParseUtils.TOTAL_WEIGHT:0.0 }
print "WARNING! SMPI1 generator: No SiteTypesWithWeights info specified in the OtherInfo field.\n\tSetting to default", SitesList
##print ">>>>>> SiteTypesWithWeightsDic", SiteTypesWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
try:
SitesWithWeightsDic = \
AIParseUtils.readStringWithWeightsList(
WLUnit['otherinfo']['SitesWithWeights'],
DefaultWeight = 1.0
)
except KeyError:
SitesWithWeightsDic = {AIParseUtils.VALUES:[], AIParseUtils.TOTAL_WEIGHT:0.0 }
print "WARNING! SMPI1 generator: No SitesWithWeights info specified in the OtherInfo field.\n\tSetting to default", SitesList
##print ">>>>>> SitesWithWeightsDic", SitesWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
else:
raise Exception, "SMPI1 generator: No OtherInfo data! Expected at least ComponentSize,NJobs,NComponents fields."
UnitsDic = {}
UnitsDic['info'] = {}
UnitsDic['jobs'] = {}
#-- init start time
try:
StartAt = AIParseUtils.readInt(WLUnit['otherinfo']['StartAt'], DefaultValue = 0)
StartAt = StartAt * 1000 # the time is given is seconds
except KeyError:
StartAt = 0
SMPI1_LastRunTime = StartAt
if 'FirstJobIndex' in WLUnit:
JobIndex = int(WLUnit['FirstJobIndex'])
else:
JobIndex = 0
WLUnitMultiplicity = int(WLUnit['multiplicity'])
if bGenerateRandom > 0 :
index = 0
while index < WLUnitMultiplicity:
# 1. generate a random site type (unordered or ordered)
SiteType = AIRandomUtils.getRandomWeightedListElement(
SiteTypesWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
# 2. select the number of CPU for this job
#TODO: repeat NCPUs selection until App can run on that many CPUs
NCPUs = AIRandomUtils.getRandomWeightedListElement(
TotalCPUsWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
# 3. select the number of components and try to create subcomponents,
# until successfully matching restrictions
WhileCondition = 1
NTries = 100
while WhileCondition != 0 and NTries > 0:
NTries = NTries - 1
NComponents = AIRandomUtils.getRandomWeightedListElement(
NComponentsWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
if NComponents > NCPUs:
continue # impossible match, just retry
NCPUsPerComponent = NCPUs / NComponents
# ensure restrictions are met
if NCPUsPerComponent < MinComponentSize or NCPUsPerComponent > MaxComponentSize:
continue
NRemainingComponents = NCPUs - NCPUsPerComponent * NComponents
if NRemainingComponents > 0 and EqualCPUsPerComponent == 1:
continue
# >= -> including the extra CPU for some components
if NRemainingComponents > 0 and NCPUsPerComponent == MaxComponentSize:
continue
PreComponentsList = []
for ComponentID in xrange(NComponents):
# cpu count
ToAssignHere = NCPUsPerComponent
if ComponentID < NRemainingComponents: # assign the extra CPUs, round-robin
ToAssignHere = ToAssignHere + 1
# site location
if SiteType == 'ordered':
Site = AIRandomUtils.getRandomWeightedListElement(
SitesWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
else:
Site = '*'
# create pre-component information
PreComponent = {}
PreComponent['id'] = ComponentID
PreComponent['count'] = ToAssignHere
PreComponent['location'] = Site
# add pre-component
PreComponentsList.append(PreComponent)
WhileCondition = 0
if WhileCondition == 1: # failed to generate in 100 tries
continue # try again for the same job
#--- generate rnd job
#-- generate job info
UnitsDic['info'][index] = \
generateJobInfo( UnitDir, UnitID, JobIndex, \
WLUnit, SubmitDurationMS, Submitter )
#-- generate job components
UnitsDic['jobs'][index] = \
generateJobComponents( UnitDir, UnitID, JobIndex, WLUnit, PreComponentsList, \
Application, AppBaseDir, \
bGenerateRandom )
index = index + 1
JobIndex = JobIndex + 1
WLUnit['generatedjobs'] = index
else:
index = 0
for Size in SMPI1Component.SMPI1_ParKSuperSizes:
for N in SMPI1Component.SMPI1_ParSupersteps:
for M in SMPI1Component.SMPI1_ParMemoryKItems:
for S in SMPI1Component.SMPI1_ParMemoryElementsPerItem:
for X in SMPI1Component.SMPI1_ParXChangeElementsPerStep:
for C in SMPI1Component.SMPI1_ParComputationPerMemoryItem:
raise Exception, "ERROR! SMPI1 is not finished!"
#--- generate wall time once for all components of this job
MaxWallTime = AIRandomUtils.getRandomListElement( SMPI1Component.SMPI1_RunTimeInSeconds )
#-- generate a random site type (unordered, ordered, ...)
SiteType = AIRandomUtils.getRandomWeightedListElement(
SiteTypesWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
#-- generate job info
UnitsDic['info'][index] = \
generateJobInfo( UnitDir, UnitID, JobIndex, \
WLUnit, SubmitDurationMS )
#-- generate job components
UnitsDic['jobs'][index] = \
generateJobComponents( UnitDir, UnitID, JobIndex, WLUnit, \
bGenerateRandom, SiteType, Size, \
N, M, S, X, C, MaxWallTime )
index = index + 1
JobIndex = JobIndex + 1
#-- set generated jobs
WLUnit['generatedjobs'] = index
return UnitsDic
def generateWorkload( UnitDir, UnitID, WLUnit, SubmitDurationMS, bGenerateRandom = 1 ):
global IBIS_LastRunTime
#print WLUnit['otherinfo']
#{'RandomWorkload': 'true', 'NComponents': '1-8', 'NJobs': '50', 'ComponentSize':'4-8'}
if bGenerateRandom > 0:
print "Random workload"
else:
print "Full workload"
if 'otherinfo' in WLUnit.keys():
try:
##print ">>>>>> NJobs"
NJobs = AIParseUtils.readInt(WLUnit['otherinfo']['NJobs'], DefaultValue = 0)
except KeyError:
NJobs = 0
if bGenerateRandom > 0: #-- NJobs required if random, optional otherwise
raise Exception, "IBIS generator: No NJobs info specified in the OtherInfo field."
try:
NComponentsWithWeightsDic = \
AIParseUtils.readIntWithWeightsList(
WLUnit['otherinfo']['NComponentsWithWeights'],
DefaultWeight = 1.0
)
# NComponentsWithWeightsDic['Values']; NComponentsWithWeightsDic['TotalWeight']
except KeyError:
raise Exception, "IBIS generator: Invalid NComponentsWithWeights info specified in the OtherInfo field."
##print ">>>>>> NComponentsWithWeightsDic", NComponentsWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
try:
TotalCPUsWithWeightsDic = \
AIParseUtils.readIntWithWeightsList(
WLUnit['otherinfo']['TotalCPUsWithWeights'],
DefaultWeight = 1.0
)
except KeyError:
raise Exception, "IBIS generator: Invalid TotalCPUsWithWeights info specified in the OtherInfo field."
##print ">>>>>> TotalCPUsWithWeightsDic", TotalCPUsWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
try:
MinComponentSize = AIParseUtils.readInt(WLUnit['otherinfo']['MinComponentSize'], DefaultValue = 0)
except KeyError:
MinComponentSize = 1
#raise Exception, "IBIS generator: No MinComponentSize info specified in the OtherInfo field."
try:
MaxComponentSize = AIParseUtils.readInt(WLUnit['otherinfo']['MaxComponentSize'], DefaultValue = 0)
except KeyError:
MaxComponentSize = 10000000
#raise Exception, "IBIS generator: No MaxComponentSize info specified in the OtherInfo field."
try:
EqualCPUsPerComponent = AIParseUtils.readBoolean(WLUnit['otherinfo']['IbisAppsDir'])
except KeyError:
EqualCPUsPerComponent = 0
#raise Exception, "IBIS generator: No EqualCPUsPerComponent info specified in the OtherInfo field. Setting to default " + AIParseUtils.IntToBooleanDic[EqualCPUsPerComponent] + '.'
try:
IbisAppsDir = WLUnit['otherinfo']['IbisAppsDir']
except KeyError:
raise Exception, "IBIS generator: No IbisAppsDir info specified in the OtherInfo field."
try:
IbisLibDir = WLUnit['otherinfo']['IbisLibDir']
except KeyError:
raise Exception, "IBIS generator: No IbisLibDir info specified in the OtherInfo field."
try:
JavaHomeDir = WLUnit['otherinfo']['JavaHomeDir']
except KeyError:
raise Exception, "IBIS generator: No JavaHomeDir info specified in the OtherInfo field."
try:
AppsWithWeightsDic = \
AIParseUtils.readStringWithWeightsList(
WLUnit['otherinfo']['AppsWithWeights'],
DefaultWeight = 1.0
)
except KeyError:
raise Exception, "IBIS generator: Invalid AppsWithWeights info specified in the OtherInfo field."
###print ">>>>>> AppsWithWeightsDic", AppsWithWeightsDic#[AIParseUtils.TOTAL_WEIGHT]
try:
Submitter = WLUnit['otherinfo']['Submitter']
except KeyError:
Submitter = 'grunner'
print "WARNING! IBIS generator: No Submitter info specified in the OtherInfo field.\n\tSetting to default", Submitter
try:
SiteTypesWithWeightsDic = \
AIParseUtils.readStringWithWeightsList(
WLUnit['otherinfo']['SiteTypesWithWeights'],
DefaultWeight = 1.0
)
except KeyError:
SiteTypesWithWeights = {AIParseUtils.VALUES:[], AIParseUtils.TOTAL_WEIGHT:0.0 }
print "WARNING! IBIS generator: No SiteTypesWithWeights info specified in the OtherInfo field.\n\tSetting to default", SitesList
##print ">>>>>> SiteTypesWithWeightsDic", SiteTypesWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
try:
SitesWithWeightsDic = \
AIParseUtils.readStringWithWeightsList(
WLUnit['otherinfo']['SitesWithWeights'],
DefaultWeight = 1.0
)
except KeyError:
SitesWithWeightsDic = {AIParseUtils.VALUES:[], AIParseUtils.TOTAL_WEIGHT:0.0 }
print "WARNING! IBIS generator: No SitesWithWeights info specified in the OtherInfo field.\n\tSetting to default", SitesList
##print ">>>>>> SitesWithWeightsDic", SitesWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
try:
JavaMaxMemoryLimitMB = AIParseUtils.readInt(WLUnit['otherinfo']['JavaMaxMemoryLimitMB'], 10000)
except KeyError:
JavaMaxMemoryLimitMB = 10000
print "WARNING! IBIS generator: No JavaMaxMemoryLimitMB info specified in the OtherInfo field.\n\tSetting to default", JavaMaxMemoryLimitMB, "MB"
try:
SelectionMethodName = WLUnit['otherinfo']['SelectionMethod']
except KeyError:
SelectionMethodName = 'R'
raise Exception, "IBIS generator: No SelectionMethod info specified in the OtherInfo field."
else:
raise Exception, "IBIS generator: No OtherInfo data! Expected at least ComponentSize,NJobs,NComponents fields."
UnitsDic = {}
UnitsDic['info'] = {}
UnitsDic['jobs'] = {}
#-- init start time
try:
StartAt = AIParseUtils.readInt(WLUnit['otherinfo']['StartAt'], DefaultValue = 0)
StartAt = StartAt * 1000 # the time is given is seconds
except KeyError:
StartAt = 0
IBIS_LastRunTime = StartAt
WLUnitMultiplicity = int(WLUnit['multiplicity'])
if bGenerateRandom > 0:
#--- generate random
OneIBISSelectionMethod = IBISSelectionMethod( AppsWithWeightsDic )
OneIBISSelectionMethod.setSelectionMethodByName( SelectionMethodName )
JobIndex = 0
if WLUnitMultiplicity > 1:
NJobs = NJobs * WLUnitMultiplicity
while JobIndex < NJobs:
# 1. select application type
## Application = AIRandomUtils.getRandomWeightedListElement(
## AppsWithWeightsDic,
## ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
## )
Application = OneIBISSelectionMethod.getNextValue()
if Application.find('*') >= 0:
print "IBIS generator: Expanded", Application, "to",
Application = ExpandApplication(Application)
print Application
# 2. select the number of CPU for this job
#TODO: repeat NCPUs selection until App can run on that many CPUs
NCPUs = AIRandomUtils.getRandomWeightedListElement(
TotalCPUsWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
# 3. select the number of components and try to create subcomponents,
# until successfully matching restrictions
WhileCondition = 1
NTries = 100
while WhileCondition != 0 and NTries > 0:
NTries = NTries - 1
NComponents = AIRandomUtils.getRandomWeightedListElement(
NComponentsWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
if NComponents > NCPUs:
continue # impossible match, just retry
NCPUsPerComponent = NCPUs / NComponents
# ensure restrictions are met
if NCPUsPerComponent < MinComponentSize or NCPUsPerComponent > MaxComponentSize:
continue
NRemainingComponents = NCPUs - NCPUsPerComponent * NComponents
if NRemainingComponents > 0 and EqualCPUsPerComponent == 1:
continue
# >= -> including the extra CPU for some components
if NRemainingComponents > 0 and NCPUsPerComponent == MaxComponentSize:
continue
SiteType = AIRandomUtils.getRandomWeightedListElement(
SiteTypesWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
PreComponentsList = []
for ComponentID in xrange(NComponents):
# cpu count
ToAssignHere = NCPUsPerComponent
if ComponentID < NRemainingComponents: # assign the extra CPUs, round-robin
ToAssignHere = ToAssignHere + 1
# site location
if SiteType == 'ordered':
Site = AIRandomUtils.getRandomWeightedListElement(
SitesWithWeightsDic,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
else:
Site = '*'
# create pre-component information
PreComponent = {}
PreComponent['id'] = ComponentID
PreComponent['count'] = ToAssignHere
PreComponent['location'] = Site
# add pre-component
PreComponentsList.append(PreComponent)
WhileCondition = 0
if WhileCondition == 1: # failed to generate in 100 tries
continue # try again for the same job
#--- generate rnd job
#-- generate rnd job: ID
CurrentUnitID = "%s-%d" % (UnitID, JobIndex % WLUnitMultiplicity)
#-- generate rnd job: index
UnitsDic['info'][JobIndex] = \
generateJobInfo( UnitDir, CurrentUnitID, JobIndex, Application, \
WLUnit, SubmitDurationMS, Submitter )
#-- generate rnd job: components
UnitsDic['jobs'][JobIndex] = \
generateJobComponents(
UnitDir, CurrentUnitID, JobIndex, WLUnit,
PreComponentsList, JavaHomeDir, IbisAppsDir, IbisLibDir,
JavaMaxMemoryLimitMB, Application, NCPUs
)
#-- be more descriptive
UnitsDic['info'][JobIndex]['description'] = \
UnitsDic['info'][JobIndex]['description'] + \
" Components: %d. NCPUs: %d." % (len(PreComponentsList), NCPUs)
#--- done generating
if UnitsDic['jobs'][JobIndex] == None:
NJobs = NJobs - 1 #-- don't ack job, but make sure total no of jobs is decreased
else:
print "IBIS generator: Generated a job of type", Application,
print "with", len(PreComponentsList), "components/",
print NCPUs, "CPUs, in", UnitsDic['info'][JobIndex]['jdf']
JobIndex = JobIndex + 1 #-- ack job
WLUnit['generatedjobs'] = JobIndex
else:
print "Complete workload"
#--- generate all
JobIndex = 0
RepetitionIndex = 0
while RepetitionIndex < WLUnitMultiplicity:
for ComponentSize in ComponentSizeList:
for NumComponents in NumComponentsList:
CurrentUnitID = "%s-%d" % (UnitID, JobIndex % WLUnitMultiplicity)
UnitsDic['info'][JobIndex] = \
generateJobInfo( UnitDir, CurrentUnitID, JobIndex, \
WLUnit, SubmitDurationMS, Submitter )
UnitsDic['jobs'][JobIndex] = \
generateJobComponents( UnitDir, CurrentUnitID, JobIndex, WLUnit, \
NumComponents, ComponentSize, Application, NCPUs )
JobIndex = JobIndex + 1
RepetitionIndex = RepetitionIndex + 1
if JobIndex > NJobs and NJobs > 0:
print "Generated %d jobs and stopped because of the NJobs limit."
break
WLUnit['generatedjobs'] = JobIndex
#raise Exception, "IBIS generator: Not implemented yet"
return UnitsDic
RunTime = IBIS_LastRunTime + WLUnit['arrivaltimefunc']( Name, ParamsList )
IBIS_LastRunTime = RunTime
InfoDic['runTime'] = "%.3f" % RunTime
except AIRandomUtils.AIRandomError, e:
print "AIRandomUtils.AIRandomError", e
InfoDic['runTime'] = None
except Exception, e:
print "ERROR!", e, "Name", Name, "Params", ParamsList
InfoDic['runTime'] = None
if (InfoDic['runTime'] == None) or \
(float(InfoDic['runTime']) < 0) or (float(InfoDic['runTime']) > SubmitDurationMS):
# all workload units are started within at most SubmitDurationMS miliseconds
print "Wrong arrival time of", InfoDic['runTime'], "generated by", Name, ParamsList
print "Resetting to default U(0,%d)" % SubmitDurationMS
InfoDic['runTime'] = str(AIRandomUtils.getRandomInt(0, SubmitDurationMS))
return InfoDic
def generateJobComponents( UnitDir, UnitID, JobIndex, WLUnit, \
PreComponentsList, JavaHomeDir, IbisAppsDir, IbisLibDir, \
JavaMaxMemoryLimitMB, Application, NCPUs):
# minutes: 3, 15, 30, 700
RunTimeInMinutesList = [ '3', '15', '30', '700' ]
NumComponents = len(PreComponentsList)
ComponentsList = []
#--- generate wall time once for all components of this job
)
GraphData['NConnect'] = \
AIRandomUtils.getRandomWeightedListElement(
Params['TaskSamePred']['NConnectWithWeights'],
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
except Exception, e:
print "Could not find a definition for", getRandomGraphTypeName(GraphType), "parameter", e
continue
GraphName = GraphName + ' NTasks=' + str(GraphData['NTasks']) + ' NConnect=' + str(GraphData['NConnect'])
elif GraphType == RandomGraphType.LayerSameProb:
try:
GraphData['NLayers'] = \
AIRandomUtils.getRandomWeightedListElement(
Params['LayerSameProb']['NLayersWithWeights'],
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
GraphData['NLayerTasks'] = \
AIRandomUtils.getRandomWeightedListElement(
Params['LayerSameProb']['NLayerTasksWithWeights'],
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
GraphData['p'] = \
AIRandomUtils.getRandomWeightedListElement(
Params['LayerSameProb']['pWithWeights'],
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
except Exception, e:
print "Could not find a definition for", getRandomGraphTypeName(GraphType), "parameter", e
continue
GraphName = GraphName + ' NLayers=' + str(GraphData['NLayers']) + ' NLayerTasks=' + str(GraphData['NLayerTasks']) + ' p=' + str(GraphData['p'])
def generateRandomStructures(WLOutDir, WLUnitID, WLUnit, SubmitDurationMS):
#--- params read
##print ">>> WLUnit=", WLUnit
try:
NCompositeItems = AIParseUtils.readInt(WLUnit['otherinfo']['NCompositeItems'], 1)
except:
NCompositeItems = 1
try:
GraphTypesWithWeights = \
AIParseUtils.readStringWithWeightsList(
WLUnit['otherinfo']['GraphTypesWithWeights'],
DefaultWeight = 1.0, ItemSeparator = ';'
)
## ValidGraphTypesWithWeights = {}
## for GraphType in GraphTypesWithWeights[AIParseUtils.VALUES]:
## if GraphTypes not in RandomGraphType.Names:
## print "WARNING!", GraphTypes, "is not a valid GraphType. Skipping it."
## else:
## ValidGraphTypesWithWeights[GraphTypes] = GraphTypesWithWeights[AIParseUtils.VALUES][GraphTypes]
## GraphTypesWithWeights[AIParseUtils.VALUES] = ValidGraphTypesWithWeights
except:
print "ERROR! No GraphTypesWithWeights info specified in the OtherInfo field.\n\tQuitting!"
return None
##print ">>> GraphTypesWithWeights=", GraphTypesWithWeights
STGFileNamesList = []
try:
STGFilters = WLUnit['otherinfo']['STGFiles']
STGFilterList = STGFilters.split(',')
for Filter in STGFilterList:
CurrentSTGList = glob.glob(Filter)
for STGFileName in CurrentSTGList:
if STGFileName not in STGFileNamesList:
STGFileNamesList.append(STGFileName)
except:
STGFileNamesList = []
pass
TextGraphFileNamesList = []
try:
TextGraphFilters = WLUnit['otherinfo']['TextGraphFiles']
TextGraphFilterList = TextGraphFilters.split(',')
for Filter in TextGraphFilterList:
CurrentTextGraphList = glob.glob(Filter)
for TextGraphFileName in CurrentTextGraphList:
if TextGraphFileName not in TextGraphFileNamesList:
TextGraphFileNamesList.append(TextGraphFileName)
#print "#### appending: " + TextGraphFileName
except:
TextGraphFileNamesList = []
#print "### TextGraph exception"
#print traceback.print_exc()
pass
##print ">>>", "before Params = readAllParams( WLUnit )"
Params = readAllParams( WLUnit )
##print ">>>", "after Params = readAllParams( WLUnit )"
#--- init
ListOfGraphs = []
#--- generate all composite structures
index = 0
maxIndex = NCompositeItems
while index < maxIndex:
#--- generate one composite structure
GraphID = WLUnitID + ("-%d_graph" % index)
#print("### Graph ID initial: " + GraphID)
#print ("### WLUnit ID initial: " + WLUnitID)
GraphType = AIRandomUtils.getRandomWeightedListElement(
GraphTypesWithWeights,
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
GraphType = RandomGraphType.Names[GraphType]
#print ">>>", "Selected", GraphType, "from", GraphTypesWithWeights
GraphName = getRandomGraphTypeName(GraphType)
GraphData = {}
if GraphType == RandomGraphType.FromSTG:
pass
elif GraphType == RandomGraphType.TaskSameProb:
try:
GraphData['NTasks'] = \
AIRandomUtils.getRandomWeightedListElement(
Params['TaskSameProb']['NTasksWithWeights'],
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
GraphData['p'] = \
AIRandomUtils.getRandomWeightedListElement(
Params['TaskSameProb']['pWithWeights'],
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
except Exception, e:
print "Could not find a definition for", getRandomGraphTypeName(GraphType), "parameter", e
continue
GraphName = GraphName + ' NTasks=' + str(GraphData['NTasks']) + ' p=' + str(GraphData['p'])
elif GraphType == RandomGraphType.TaskSamePred:
try:
GraphData['NTasks'] = \
AIRandomUtils.getRandomWeightedListElement(
Params['TaskSamePred']['NTasksWithWeights'],
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
GraphData['NConnect'] = \
AIRandomUtils.getRandomWeightedListElement(
Params['TaskSamePred']['NConnectWithWeights'],
ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT
)
except Exception, e:
print "Could not find a definition for", getRandomGraphTypeName(GraphType), "parameter", e
continue
except AIRandomUtils.AIRandomError, e:
print "AIRandomUtils.AIRandomError", e
InfoDic["runTime"] = None
except Exception, e:
print "ERROR!", e, "Name", Name, "Params", ParamsList
InfoDic["runTime"] = None
if (
(InfoDic["runTime"] == None)
or (float(InfoDic["runTime"]) < 0)
or (float(InfoDic["runTime"]) > SubmitDurationMS)
):
# all workload units are started within at most SubmitDurationMS miliseconds
print "Wrong arrival time of", InfoDic["runTime"], "generated by", Name, ParamsList
print "Resetting to default U(0,%d)" % SubmitDurationMS
InfoDic["runTime"] = str(AIRandomUtils.getRandomInt(0, SubmitDurationMS))
return InfoDic
def generateJobComponents(UnitDir, UnitID, JobIndex, WLUnit, NumComponents, ComponentSize, Application):
# minutes: 3, 15, 30, 700
RunTimeInMinutesList = ["3", "15", "30", "700"]
ComponentsList = []
# --- generate wall time once for all components of this job
MaxWallTime = AIRandomUtils.getRandomListElement(RunTimeInMinutesList)
if UnitDir[0] != "/":