def readAllParams( WLUnit ): Params = {} for ParamCategory in BPatternGraphType.Names: Params[ParamCategory] = {} try: Sequence_NTasksWithWeights = \ AIParseUtils.readIntWithWeightsList( WLUnit['otherinfo']['Params.Sequence.NTasksWithWeights'], DefaultWeight = 1.0, ItemSeparator = ',' ) Params['Sequence']['NTasksWithWeights'] = Sequence_NTasksWithWeights except: ## print traceback.print_exc() pass try: ParallelSplit_NBranchesWithWeights = \ AIParseUtils.readIntWithWeightsList( WLUnit['otherinfo']['Params.ParallelSplit.NBranchesWithWeights'], DefaultWeight = 1.0, ItemSeparator = ',' ) Params['ParallelSplit']['NBranchesWithWeights'] = ParallelSplit_NBranchesWithWeights except: ## print traceback.print_exc() pass try: ParallelSplit_MinTasksPerBranchWithWeights = \ AIParseUtils.readIntWithWeightsList( WLUnit['otherinfo']['Params.ParallelSplit.MinTasksPerBranchWithWeights'], DefaultWeight = 1.0, ItemSeparator = ',' ) Params['ParallelSplit']['MinTasksPerBranchWithWeights'] = ParallelSplit_MinTasksPerBranchWithWeights except: ## print traceback.print_exc() pass try: ParallelSplit_MaxTasksPerBranchWithWeights = \ AIParseUtils.readIntWithWeightsList( WLUnit['otherinfo']['Params.ParallelSplit.MaxTasksPerBranchWithWeights'], DefaultWeight = 1.0, ItemSeparator = ',' ) Params['ParallelSplit']['MaxTasksPerBranchWithWeights'] = ParallelSplit_MaxTasksPerBranchWithWeights except: ## print traceback.print_exc() pass return Params
def readAllParams( WLUnit ): Params = {} for ParamCategory in RandomGraphType.Names: Params[ParamCategory] = {} try: TaskSameProb_NTasksWithWeights = \ AIParseUtils.readIntWithWeightsList( WLUnit['otherinfo']['Params.TaskSameProb.NTasksWithWeights'], DefaultWeight = 1.0, ItemSeparator = ',' ) Params['TaskSameProb']['NTasksWithWeights'] = TaskSameProb_NTasksWithWeights except: ## print traceback.print_exc() pass try: TaskSameProb_pWithWeights = \ AIParseUtils.readFloatWithWeightsList( WLUnit['otherinfo']['Params.TaskSameProb.pWithWeights'], DefaultWeight = 1.0, ItemSeparator = ',' ) Params['TaskSameProb']['pWithWeights'] = TaskSameProb_pWithWeights except: ## print traceback.print_exc() pass try: TaskSamePred_NTasksWithWeights = \ AIParseUtils.readIntWithWeightsList( WLUnit['otherinfo']['Params.TaskSamePred.NTasksWithWeights'], DefaultWeight = 1.0, ItemSeparator = ',' ) Params['TaskSamePred']['NTasksWithWeights'] = TaskSamePred_NTasksWithWeights except: ## print traceback.print_exc() pass try: TaskSamePred_NTasksWithWeights = \ AIParseUtils.readIntWithWeightsList( WLUnit['otherinfo']['Params.TaskSamePred.NConnectWithWeights'], DefaultWeight = 1.0, ItemSeparator = ',' ) Params['TaskSamePred']['NConnectWithWeights'] = TaskSamePred_NTasksWithWeights except: ## print traceback.print_exc() pass try: LayerSameProb_NLayersWithWeights = \ AIParseUtils.readTupleWithWeightsList( WLUnit['otherinfo']['Params.LayerSameProb.NLayersWithWeights'], TupleType = AIParseUtils.TupleValues.INT_VALUE, DefaultWeight = 1.0, ItemSeparator = ';', TupleItemSeparator=',' ) Params['LayerSameProb']['NLayersWithWeights'] = LayerSameProb_NLayersWithWeights except: ## print traceback.print_exc() pass try: LayerSameProb_NLayerTasksWithWeights = \ AIParseUtils.readTupleWithWeightsList( WLUnit['otherinfo']['Params.LayerSameProb.NLayerTasksWithWeights'], TupleType = AIParseUtils.TupleValues.INT_VALUE, DefaultWeight = 1.0, ItemSeparator = ';', TupleItemSeparator=',' ) Params['LayerSameProb']['NLayerTasksWithWeights'] = LayerSameProb_NLayerTasksWithWeights except: ## print traceback.print_exc() pass try: LayerSameProb_pWithWeights = \ AIParseUtils.readFloatWithWeightsList( WLUnit['otherinfo']['Params.LayerSameProb.pWithWeights'], DefaultWeight = 1.0, ItemSeparator = ',' ) Params['LayerSameProb']['pWithWeights'] = LayerSameProb_pWithWeights except: ## print traceback.print_exc() pass try: LayerSamePred_NLayersWithWeights = \ AIParseUtils.readTupleWithWeightsList( WLUnit['otherinfo']['Params.LayerSamePred.NLayersWithWeights'], TupleType = AIParseUtils.TupleValues.INT_VALUE, DefaultWeight = 1.0, ItemSeparator = ';', TupleItemSeparator=',' ) Params['LayerSamePred']['NLayersWithWeights'] = LayerSamePred_NLayersWithWeights except: ## print traceback.print_exc() pass try: LayerSamePred_NLayerTasksWithWeights = \ AIParseUtils.readTupleWithWeightsList( WLUnit['otherinfo']['Params.LayerSamePred.NLayerTasksWithWeights'], TupleType = AIParseUtils.TupleValues.INT_VALUE, DefaultWeight = 1.0, ItemSeparator = ';', TupleItemSeparator=',' ) Params['LayerSamePred']['NLayerTasksWithWeights'] = LayerSamePred_NLayerTasksWithWeights except: ## print traceback.print_exc() pass try: LayerSamePred_NTasksWithWeights = \ AIParseUtils.readIntWithWeightsList( WLUnit['otherinfo']['Params.LayerSamePred.NConnectWithWeights'], DefaultWeight = 1.0, ItemSeparator = ',' ) Params['LayerSamePred']['NConnectWithWeights'] = LayerSamePred_NTasksWithWeights except: ## print traceback.print_exc() pass return Params
def readParams( ParamsDic ): """ read params, return a SWFParams class, with new fields """ CurrentSWFParams = SWFParams() CurrentSWFParams.SWF = SWFOnlyParams() if 'otherinfo' in ParamsDic.keys(): SWFConstsInstance = SWFConsts() try: for key in ParamsDic['otherinfo']: OneList = key.split('.') if OneList[0] == 'SWF': if OneList[1] == 'File': CurrentSWFParams.SWF.File = ParamsDic['otherinfo'][key] elif OneList[1] == 'Scale': # SWF.Scale.SubmitTime=1:1 KeyID = SWFConstsInstance.getID(OneList[2]) if (KeyID is not None) and (KeyID >= 0): # save: ['SubmitTime']: (1,1) ScaleIn, ScaleOut = ParamsDic['otherinfo'][key].split(':') ScaleIn = float(ScaleIn) ScaleOut = float(ScaleOut) CurrentSWFParams.SWF.Scale[OneList[2]] = (ScaleIn, ScaleOut) elif OneList[1] == 'Filter': KeyID = SWFConstsInstance.getID(OneList[2]) ##print 'Filter', OneList[2], "KeyID=", KeyID if (KeyID is not None) and (KeyID >= 0): #SWF.Filter.JobNumber.Min=31 # save: ['JobNumber']: ('Min',31) if OneList[2] not in CurrentSWFParams.SWF.Filters: CurrentSWFParams.SWF.Filters[OneList[2]] = [] CurrentSWFParams.SWF.Filters[OneList[2]].append((OneList[3], int(ParamsDic['otherinfo'][key]))) else: # special cases ##print "Special case filter", OneList[2] # SWF.Filter.Delta.SubmitTime.Hours.Max=3 # save: ['Delta']['SubmitTime']['Hours']('Max',31) if OneList[2] == 'Delta': if 'Delta' not in CurrentSWFParams.SWF.Filters: CurrentSWFParams.SWF.Filters['Delta'] = {} KeyID = SWFConstsInstance.getID(OneList[3]) if KeyID and KeyID >= 0: if OneList[3] not in CurrentSWFParams.SWF.Filters['Delta']: CurrentSWFParams.SWF.Filters['Delta'][OneList[3]] = {} CurrentSWFParams.SWF.Filters['Delta'][OneList[3]][OneList[4]] = (OneList[5], int(ParamsDic['otherinfo'][key])) # SWF.Filter.NJobs.Max=1000 # save: ['NJobs']: ('Max',1000) if OneList[2] == 'NJobs': CurrentSWFParams.SWF.Filters['NJobs'] = (OneList[3], int(ParamsDic['otherinfo'][key])) except Exception, e: print "Exception", e print traceback.print_exc() raise Exception, "SWF generator: could not decode " + str(key) + '.' if CurrentSWFParams.SWF.File is None: raise Exception, "SWF generator: No SWF.File field in OtherInfo (cannot read an unknown file)." # UseArrivalTimeDistribution=false try: CurrentSWFParams.UseArrivalTimeDistribution = AIParseUtils.readBoolean(ParamsDic['otherinfo']['UseArrivalTimeDistribution']) except KeyError: CurrentSWFParams.UseArrivalTimeDistribution = 0 #Application=swf #AppBaseDir=/tmp try: CurrentSWFParams.AppBaseDir = ParamsDic['otherinfo']['AppBaseDir'] except KeyError: CurrentSWFParams.AppBaseDir = '/tmp' #raise Exception, "SMPI1 generator: No AppBaseDir info specified in the OtherInfo field." print "WARNING! SWF generator: No AppBaseDir info specified in the OtherInfo field.", "Assuming default:", AppBaseDir try: CurrentSWFParams.Application = ParamsDic['otherinfo']['Application'] except KeyError: CurrentSWFParams.Application = 'swf' #raise Exception, "SMPI1 generator: No AppBaseDir info specified in the OtherInfo field." print "WARNING! SWF generator: No Application info specified in the OtherInfo field.", "Assuming default:", Application try: CurrentSWFParams.Submitter = ParamsDic['otherinfo']['Submitter'] except KeyError: CurrentSWFParams.Submitter = 'krunner -l DEBUG -g -e -o -f ${JDF}' print "WARNING! SWF generator: No Submitter info specified in the OtherInfo field.", "Assuming default", Submitter try: CurrentSWFParams.NComponentsWithWeightsDic = \ AIParseUtils.readIntWithWeightsList( ParamsDic['otherinfo']['NComponentsWithWeights'], DefaultWeight = 1.0 ) # NComponentsWithWeightsDic['Values']; NComponentsWithWeightsDic['TotalWeight'] except KeyError: raise Exception, "SWF generator: Invalid NComponentsWithWeights info specified in the OtherInfo field." ##print ">>>>>> NComponentsWithWeightsDic", NComponentsWithWeightsDic[AIParseUtils.TOTAL_WEIGHT] try: CurrentSWFParams.MinComponentSize = AIParseUtils.readInt(ParamsDic['otherinfo']['MinComponentSize'], DefaultValue = 0) except KeyError: CurrentSWFParams.MinComponentSize = 1 print "WARNING! SWF generator: No MinComponentSize info specified in the OtherInfo field.", "Assuming default", CurrentSWFParams.MinComponentSize #raise Exception, "SWF generator: No MinComponentSize info specified in the OtherInfo field." try: CurrentSWFParams.DelayBetweenUnits = AIParseUtils.readInt(ParamsDic['otherinfo']['DelayBetweenUnits'], DefaultValue = 0) except KeyError: CurrentSWFParams.DelayBetweenUnits = 1 print "WARNING! SWF generator: No DelayBetweenUnits info specified in the OtherInfo field.", "Assuming default", CurrentSWFParams.DelayBetweenUnits try: CurrentSWFParams.MaxWallTime = AIParseUtils.readInt(ParamsDic['otherinfo']['MaxWallTime'], DefaultValue = 1000000) except KeyError: CurrentSWFParams.MaxWallTime = 1 print "WARNING! SWF generator: No DelayBetweenUnits info specified in the OtherInfo field.", "Assuming default", CurrentSWFParams.MaxWallTime try: CurrentSWFParams.MaxComponentSize = AIParseUtils.readInt(ParamsDic['otherinfo']['MaxComponentSize'], DefaultValue = 0) except KeyError: CurrentSWFParams.MaxComponentSize = 10000000 print "WARNING! SWF generator: No MaxComponentSize info specified in the OtherInfo field.", "Assuming default", CurrentSWFParams.MaxComponentSize #raise Exception, "SWF generator: No MaxComponentSize info specified in the OtherInfo field." try: CurrentSWFParams.EqualCPUsPerComponent = AIParseUtils.readBoolean(ParamsDic['otherinfo']['SWFAppsDir']) except KeyError: CurrentSWFParams.EqualCPUsPerComponent = 0 #raise Exception, "SWF generator: No EqualCPUsPerComponent info specified in the OtherInfo field. Setting to default " + AIParseUtils.IntToBooleanDic[EqualCPUsPerComponent] + '.' try: CurrentSWFParams.SiteTypesWithWeightsDic = \ AIParseUtils.readStringWithWeightsList( ParamsDic['otherinfo']['SiteTypesWithWeights'], DefaultWeight = 1.0 ) except KeyError: CurrentSWFParams.SiteTypesWithWeights = {AIParseUtils.VALUES:[], AIParseUtils.TOTAL_WEIGHT:0.0 } print "WARNING! SWF generator: No SiteTypesWithWeights info specified in the OtherInfo field.\n\tSetting to default..." ##print ">>>>>> SiteTypesWithWeightsDic", SiteTypesWithWeightsDic[AIParseUtils.TOTAL_WEIGHT] try: CurrentSWFParams.SitesWithWeightsDic = \ AIParseUtils.readStringWithWeightsList( ParamsDic['otherinfo']['SitesWithWeights'], DefaultWeight = 1.0 ) except KeyError: CurrentSWFParams.SitesWithWeightsDic = {AIParseUtils.VALUES:[], AIParseUtils.TOTAL_WEIGHT:0.0 } print "WARNING! SWF generator: No SitesWithWeights info specified in the OtherInfo field.\n\tSetting to default..."
def generateWorkloadUnit( UnitDir, UnitID, WLUnit, SubmitDurationMS, bGenerateRandom = 1 ): """ Out: UnitsDic o A dictionary with keys 'info' and 'jobs'. Notes: o UnitsDic['info'] contains a dictionary of jobs info, indexed with an integer counter o Each job info contains at least the keys name, description, jdf, submitCommand, runTime o UnitsDic['jobs'] contains a dictionary of jobs data, indexed with an integer counter o Each job data contains the list of components of that job o Each component in the list is a dictionary with at least the following fields: executable, stdout, stderr, name, description, directory, maxWallTime, arguments, env, stagein, stageout See also: utils/WLDocHandlers.WLSubmitJobKeys this_file.SMPI1Component.generateComponent """ global SMPI1_LastRunTime SMPI1Component.SMPI1_Exe = os.path.join( "/tmp", "smpi1t" ) SMPI1Component.SMPI1_ParKSizes = [ "0", "32", "128", "512", "1024" ] SMPI1Component.SMPI1_ParKSuperSizes = [ "0", "16", "32", "64", "128", "256" ] SMPI1Component.SMPI1_ParSupersteps = [ "1", "2", "5", "10", "20", "50", "100" ] SMPI1Component.SMPI1_ParMemoryKItems = [ "10", "25", "50", "100", "250", "500", "1000", "5000" ] SMPI1Component.SMPI1_ParMemoryElementsPerItem = [ "3", "4", "10", "100", "500", "1000" ] SMPI1Component.SMPI1_ParXChangeElementsPerStep = [ "100", "500", "1000", "10000", "50000" ] SMPI1Component.SMPI1_ParComputationPerMemoryItem = [ "2", "10", "100", "1000" ] SMPI1Component.SMPI1_RunTimeInMinutes = [ "2", "5", "10", "15" ] if 'otherinfo' in WLUnit.keys(): try: OneString = WLUnit['otherinfo']['Exe'] SMPI1Component.SMPI1_Exe = OneString except KeyError: pass try: OneIntList = AIParseUtils.readIntList( Text = WLUnit['otherinfo']['ParKSizes'], ItemSeparator = ',' ) SMPI1Component.SMPI1_ParKSizes = OneIntList except KeyError: pass try: OneIntList = AIParseUtils.readIntList( Text = WLUnit['otherinfo']['ParKSuperSizes'], ItemSeparator = ',' ) SMPI1Component.SMPI1_ParKSuperSizes = OneIntList except KeyError: pass try: OneIntList = AIParseUtils.readIntList( Text = WLUnit['otherinfo']['ParSupersteps'], ItemSeparator = ',' ) SMPI1Component.SMPI1_ParSupersteps = OneIntList except KeyError: pass try: OneIntList = AIParseUtils.readIntList( Text = WLUnit['otherinfo']['ParMemoryKItems'], ItemSeparator = ',' ) SMPI1Component.SMPI1_ParMemoryKItems = OneIntList except KeyError: pass try: OneIntList = AIParseUtils.readIntList( Text = WLUnit['otherinfo']['ParMemoryElementsPerItem'], ItemSeparator = ',' ) SMPI1Component.SMPI1_ParMemoryElementsPerItem = OneIntList except KeyError: pass try: OneIntList = AIParseUtils.readIntList( Text = WLUnit['otherinfo']['ParXChangeElementsPerStep'], ItemSeparator = ',' ) SMPI1Component.SMPI1_ParXChangeElementsPerStep = OneIntList except KeyError: pass try: OneIntList = AIParseUtils.readIntList( Text = WLUnit['otherinfo']['ParComputationPerMemoryItem'], ItemSeparator = ',' ) SMPI1Component.SMPI1_ParComputationPerMemoryItem = OneIntList except KeyError: pass try: OneIntList = AIParseUtils.readIntList( Text = WLUnit['otherinfo']['RunTimeInMinutes'], ItemSeparator = ',' ) SMPI1Component.SMPI1_RunTimeInMinutes = OneIntList except KeyError: pass try: NComponentsWithWeightsDic = \ AIParseUtils.readIntWithWeightsList( WLUnit['otherinfo']['NComponentsWithWeights'], DefaultWeight = 1.0 ) # NComponentsWithWeightsDic['Values']; NComponentsWithWeightsDic['TotalWeight'] except KeyError: raise Exception, "SMPI1 generator: Invalid NComponentsWithWeights info specified in the OtherInfo field." ##print ">>>>>> NComponentsWithWeightsDic", NComponentsWithWeightsDic[AIParseUtils.TOTAL_WEIGHT] try: TotalCPUsWithWeightsDic = \ AIParseUtils.readIntWithWeightsList( WLUnit['otherinfo']['TotalCPUsWithWeights'], DefaultWeight = 1.0 ) except KeyError: raise Exception, "SMPI1 generator: Invalid TotalCPUsWithWeights info specified in the OtherInfo field." ##print ">>>>>> TotalCPUsWithWeightsDic", TotalCPUsWithWeightsDic[AIParseUtils.TOTAL_WEIGHT] try: MinComponentSize = AIParseUtils.readInt(WLUnit['otherinfo']['MinComponentSize'], DefaultValue = 0) except KeyError: MinComponentSize = 1 #raise Exception, "SMPI1 generator: No MinComponentSize info specified in the OtherInfo field." try: MaxComponentSize = AIParseUtils.readInt(WLUnit['otherinfo']['MaxComponentSize'], DefaultValue = 0) except KeyError: MaxComponentSize = 10000000 #raise Exception, "SMPI1 generator: No MaxComponentSize info specified in the OtherInfo field." try: EqualCPUsPerComponent = AIParseUtils.readBoolean(WLUnit['otherinfo']['SMPI1AppsDir']) except KeyError: EqualCPUsPerComponent = 0 #raise Exception, "SMPI1 generator: No EqualCPUsPerComponent info specified in the OtherInfo field. Setting to default " + AIParseUtils.IntToBooleanDic[EqualCPUsPerComponent] + '.' try: AppBaseDir = WLUnit['otherinfo']['AppBaseDir'] except KeyError: AppBaseDir = '/tmp' #raise Exception, "SMPI1 generator: No AppBaseDir info specified in the OtherInfo field." print "WARNING! SMPI1 generator: No AppBaseDir info specified in the OtherInfo field.", "Assuming default:", AppBaseDir try: Application = WLUnit['otherinfo']['Application'] except KeyError: Application = 'smpi1t-gm' #raise Exception, "SMPI1 generator: No AppBaseDir info specified in the OtherInfo field." print "WARNING! SMPI1 generator: No Application info specified in the OtherInfo field.", "Assuming default:", Application try: Submitter = WLUnit['otherinfo']['Submitter'] except KeyError: Submitter = 'krunner -l DEBUG -g -e -o -f ${JDF}' print "WARNING! SMPI1 generator: No Submitter info specified in the OtherInfo field.", "Assuming default", Submitter try: SiteTypesWithWeightsDic = \ AIParseUtils.readStringWithWeightsList( WLUnit['otherinfo']['SiteTypesWithWeights'], DefaultWeight = 1.0 ) except KeyError: SiteTypesWithWeights = {AIParseUtils.VALUES:[], AIParseUtils.TOTAL_WEIGHT:0.0 } print "WARNING! SMPI1 generator: No SiteTypesWithWeights info specified in the OtherInfo field.\n\tSetting to default", SitesList ##print ">>>>>> SiteTypesWithWeightsDic", SiteTypesWithWeightsDic[AIParseUtils.TOTAL_WEIGHT] try: SitesWithWeightsDic = \ AIParseUtils.readStringWithWeightsList( WLUnit['otherinfo']['SitesWithWeights'], DefaultWeight = 1.0 ) except KeyError: SitesWithWeightsDic = {AIParseUtils.VALUES:[], AIParseUtils.TOTAL_WEIGHT:0.0 } print "WARNING! SMPI1 generator: No SitesWithWeights info specified in the OtherInfo field.\n\tSetting to default", SitesList ##print ">>>>>> SitesWithWeightsDic", SitesWithWeightsDic[AIParseUtils.TOTAL_WEIGHT] else: raise Exception, "SMPI1 generator: No OtherInfo data! Expected at least ComponentSize,NJobs,NComponents fields." UnitsDic = {} UnitsDic['info'] = {} UnitsDic['jobs'] = {} #-- init start time try: StartAt = AIParseUtils.readInt(WLUnit['otherinfo']['StartAt'], DefaultValue = 0) StartAt = StartAt * 1000 # the time is given is seconds except KeyError: StartAt = 0 SMPI1_LastRunTime = StartAt if 'FirstJobIndex' in WLUnit: JobIndex = int(WLUnit['FirstJobIndex']) else: JobIndex = 0 WLUnitMultiplicity = int(WLUnit['multiplicity']) if bGenerateRandom > 0 : index = 0 while index < WLUnitMultiplicity: # 1. generate a random site type (unordered or ordered) SiteType = AIRandomUtils.getRandomWeightedListElement( SiteTypesWithWeightsDic, ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT ) # 2. select the number of CPU for this job #TODO: repeat NCPUs selection until App can run on that many CPUs NCPUs = AIRandomUtils.getRandomWeightedListElement( TotalCPUsWithWeightsDic, ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT ) # 3. select the number of components and try to create subcomponents, # until successfully matching restrictions WhileCondition = 1 NTries = 100 while WhileCondition != 0 and NTries > 0: NTries = NTries - 1 NComponents = AIRandomUtils.getRandomWeightedListElement( NComponentsWithWeightsDic, ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT ) if NComponents > NCPUs: continue # impossible match, just retry NCPUsPerComponent = NCPUs / NComponents # ensure restrictions are met if NCPUsPerComponent < MinComponentSize or NCPUsPerComponent > MaxComponentSize: continue NRemainingComponents = NCPUs - NCPUsPerComponent * NComponents if NRemainingComponents > 0 and EqualCPUsPerComponent == 1: continue # >= -> including the extra CPU for some components if NRemainingComponents > 0 and NCPUsPerComponent == MaxComponentSize: continue PreComponentsList = [] for ComponentID in xrange(NComponents): # cpu count ToAssignHere = NCPUsPerComponent if ComponentID < NRemainingComponents: # assign the extra CPUs, round-robin ToAssignHere = ToAssignHere + 1 # site location if SiteType == 'ordered': Site = AIRandomUtils.getRandomWeightedListElement( SitesWithWeightsDic, ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT ) else: Site = '*' # create pre-component information PreComponent = {} PreComponent['id'] = ComponentID PreComponent['count'] = ToAssignHere PreComponent['location'] = Site # add pre-component PreComponentsList.append(PreComponent) WhileCondition = 0 if WhileCondition == 1: # failed to generate in 100 tries continue # try again for the same job #--- generate rnd job #-- generate job info UnitsDic['info'][index] = \ generateJobInfo( UnitDir, UnitID, JobIndex, \ WLUnit, SubmitDurationMS, Submitter ) #-- generate job components UnitsDic['jobs'][index] = \ generateJobComponents( UnitDir, UnitID, JobIndex, WLUnit, PreComponentsList, \ Application, AppBaseDir, \ bGenerateRandom ) index = index + 1 JobIndex = JobIndex + 1 WLUnit['generatedjobs'] = index else: index = 0 for Size in SMPI1Component.SMPI1_ParKSuperSizes: for N in SMPI1Component.SMPI1_ParSupersteps: for M in SMPI1Component.SMPI1_ParMemoryKItems: for S in SMPI1Component.SMPI1_ParMemoryElementsPerItem: for X in SMPI1Component.SMPI1_ParXChangeElementsPerStep: for C in SMPI1Component.SMPI1_ParComputationPerMemoryItem: raise Exception, "ERROR! SMPI1 is not finished!" #--- generate wall time once for all components of this job MaxWallTime = AIRandomUtils.getRandomListElement( SMPI1Component.SMPI1_RunTimeInSeconds ) #-- generate a random site type (unordered, ordered, ...) SiteType = AIRandomUtils.getRandomWeightedListElement( SiteTypesWithWeightsDic, ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT ) #-- generate job info UnitsDic['info'][index] = \ generateJobInfo( UnitDir, UnitID, JobIndex, \ WLUnit, SubmitDurationMS ) #-- generate job components UnitsDic['jobs'][index] = \ generateJobComponents( UnitDir, UnitID, JobIndex, WLUnit, \ bGenerateRandom, SiteType, Size, \ N, M, S, X, C, MaxWallTime ) index = index + 1 JobIndex = JobIndex + 1 #-- set generated jobs WLUnit['generatedjobs'] = index return UnitsDic
def generateWorkload( UnitDir, UnitID, WLUnit, SubmitDurationMS, bGenerateRandom = 1 ): global IBIS_LastRunTime #print WLUnit['otherinfo'] #{'RandomWorkload': 'true', 'NComponents': '1-8', 'NJobs': '50', 'ComponentSize':'4-8'} if bGenerateRandom > 0: print "Random workload" else: print "Full workload" if 'otherinfo' in WLUnit.keys(): try: ##print ">>>>>> NJobs" NJobs = AIParseUtils.readInt(WLUnit['otherinfo']['NJobs'], DefaultValue = 0) except KeyError: NJobs = 0 if bGenerateRandom > 0: #-- NJobs required if random, optional otherwise raise Exception, "IBIS generator: No NJobs info specified in the OtherInfo field." try: NComponentsWithWeightsDic = \ AIParseUtils.readIntWithWeightsList( WLUnit['otherinfo']['NComponentsWithWeights'], DefaultWeight = 1.0 ) # NComponentsWithWeightsDic['Values']; NComponentsWithWeightsDic['TotalWeight'] except KeyError: raise Exception, "IBIS generator: Invalid NComponentsWithWeights info specified in the OtherInfo field." ##print ">>>>>> NComponentsWithWeightsDic", NComponentsWithWeightsDic[AIParseUtils.TOTAL_WEIGHT] try: TotalCPUsWithWeightsDic = \ AIParseUtils.readIntWithWeightsList( WLUnit['otherinfo']['TotalCPUsWithWeights'], DefaultWeight = 1.0 ) except KeyError: raise Exception, "IBIS generator: Invalid TotalCPUsWithWeights info specified in the OtherInfo field." ##print ">>>>>> TotalCPUsWithWeightsDic", TotalCPUsWithWeightsDic[AIParseUtils.TOTAL_WEIGHT] try: MinComponentSize = AIParseUtils.readInt(WLUnit['otherinfo']['MinComponentSize'], DefaultValue = 0) except KeyError: MinComponentSize = 1 #raise Exception, "IBIS generator: No MinComponentSize info specified in the OtherInfo field." try: MaxComponentSize = AIParseUtils.readInt(WLUnit['otherinfo']['MaxComponentSize'], DefaultValue = 0) except KeyError: MaxComponentSize = 10000000 #raise Exception, "IBIS generator: No MaxComponentSize info specified in the OtherInfo field." try: EqualCPUsPerComponent = AIParseUtils.readBoolean(WLUnit['otherinfo']['IbisAppsDir']) except KeyError: EqualCPUsPerComponent = 0 #raise Exception, "IBIS generator: No EqualCPUsPerComponent info specified in the OtherInfo field. Setting to default " + AIParseUtils.IntToBooleanDic[EqualCPUsPerComponent] + '.' try: IbisAppsDir = WLUnit['otherinfo']['IbisAppsDir'] except KeyError: raise Exception, "IBIS generator: No IbisAppsDir info specified in the OtherInfo field." try: IbisLibDir = WLUnit['otherinfo']['IbisLibDir'] except KeyError: raise Exception, "IBIS generator: No IbisLibDir info specified in the OtherInfo field." try: JavaHomeDir = WLUnit['otherinfo']['JavaHomeDir'] except KeyError: raise Exception, "IBIS generator: No JavaHomeDir info specified in the OtherInfo field." try: AppsWithWeightsDic = \ AIParseUtils.readStringWithWeightsList( WLUnit['otherinfo']['AppsWithWeights'], DefaultWeight = 1.0 ) except KeyError: raise Exception, "IBIS generator: Invalid AppsWithWeights info specified in the OtherInfo field." ###print ">>>>>> AppsWithWeightsDic", AppsWithWeightsDic#[AIParseUtils.TOTAL_WEIGHT] try: Submitter = WLUnit['otherinfo']['Submitter'] except KeyError: Submitter = 'grunner' print "WARNING! IBIS generator: No Submitter info specified in the OtherInfo field.\n\tSetting to default", Submitter try: SiteTypesWithWeightsDic = \ AIParseUtils.readStringWithWeightsList( WLUnit['otherinfo']['SiteTypesWithWeights'], DefaultWeight = 1.0 ) except KeyError: SiteTypesWithWeights = {AIParseUtils.VALUES:[], AIParseUtils.TOTAL_WEIGHT:0.0 } print "WARNING! IBIS generator: No SiteTypesWithWeights info specified in the OtherInfo field.\n\tSetting to default", SitesList ##print ">>>>>> SiteTypesWithWeightsDic", SiteTypesWithWeightsDic[AIParseUtils.TOTAL_WEIGHT] try: SitesWithWeightsDic = \ AIParseUtils.readStringWithWeightsList( WLUnit['otherinfo']['SitesWithWeights'], DefaultWeight = 1.0 ) except KeyError: SitesWithWeightsDic = {AIParseUtils.VALUES:[], AIParseUtils.TOTAL_WEIGHT:0.0 } print "WARNING! IBIS generator: No SitesWithWeights info specified in the OtherInfo field.\n\tSetting to default", SitesList ##print ">>>>>> SitesWithWeightsDic", SitesWithWeightsDic[AIParseUtils.TOTAL_WEIGHT] try: JavaMaxMemoryLimitMB = AIParseUtils.readInt(WLUnit['otherinfo']['JavaMaxMemoryLimitMB'], 10000) except KeyError: JavaMaxMemoryLimitMB = 10000 print "WARNING! IBIS generator: No JavaMaxMemoryLimitMB info specified in the OtherInfo field.\n\tSetting to default", JavaMaxMemoryLimitMB, "MB" try: SelectionMethodName = WLUnit['otherinfo']['SelectionMethod'] except KeyError: SelectionMethodName = 'R' raise Exception, "IBIS generator: No SelectionMethod info specified in the OtherInfo field." else: raise Exception, "IBIS generator: No OtherInfo data! Expected at least ComponentSize,NJobs,NComponents fields." UnitsDic = {} UnitsDic['info'] = {} UnitsDic['jobs'] = {} #-- init start time try: StartAt = AIParseUtils.readInt(WLUnit['otherinfo']['StartAt'], DefaultValue = 0) StartAt = StartAt * 1000 # the time is given is seconds except KeyError: StartAt = 0 IBIS_LastRunTime = StartAt WLUnitMultiplicity = int(WLUnit['multiplicity']) if bGenerateRandom > 0: #--- generate random OneIBISSelectionMethod = IBISSelectionMethod( AppsWithWeightsDic ) OneIBISSelectionMethod.setSelectionMethodByName( SelectionMethodName ) JobIndex = 0 if WLUnitMultiplicity > 1: NJobs = NJobs * WLUnitMultiplicity while JobIndex < NJobs: # 1. select application type ## Application = AIRandomUtils.getRandomWeightedListElement( ## AppsWithWeightsDic, ## ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT ## ) Application = OneIBISSelectionMethod.getNextValue() if Application.find('*') >= 0: print "IBIS generator: Expanded", Application, "to", Application = ExpandApplication(Application) print Application # 2. select the number of CPU for this job #TODO: repeat NCPUs selection until App can run on that many CPUs NCPUs = AIRandomUtils.getRandomWeightedListElement( TotalCPUsWithWeightsDic, ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT ) # 3. select the number of components and try to create subcomponents, # until successfully matching restrictions WhileCondition = 1 NTries = 100 while WhileCondition != 0 and NTries > 0: NTries = NTries - 1 NComponents = AIRandomUtils.getRandomWeightedListElement( NComponentsWithWeightsDic, ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT ) if NComponents > NCPUs: continue # impossible match, just retry NCPUsPerComponent = NCPUs / NComponents # ensure restrictions are met if NCPUsPerComponent < MinComponentSize or NCPUsPerComponent > MaxComponentSize: continue NRemainingComponents = NCPUs - NCPUsPerComponent * NComponents if NRemainingComponents > 0 and EqualCPUsPerComponent == 1: continue # >= -> including the extra CPU for some components if NRemainingComponents > 0 and NCPUsPerComponent == MaxComponentSize: continue SiteType = AIRandomUtils.getRandomWeightedListElement( SiteTypesWithWeightsDic, ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT ) PreComponentsList = [] for ComponentID in xrange(NComponents): # cpu count ToAssignHere = NCPUsPerComponent if ComponentID < NRemainingComponents: # assign the extra CPUs, round-robin ToAssignHere = ToAssignHere + 1 # site location if SiteType == 'ordered': Site = AIRandomUtils.getRandomWeightedListElement( SitesWithWeightsDic, ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT ) else: Site = '*' # create pre-component information PreComponent = {} PreComponent['id'] = ComponentID PreComponent['count'] = ToAssignHere PreComponent['location'] = Site # add pre-component PreComponentsList.append(PreComponent) WhileCondition = 0 if WhileCondition == 1: # failed to generate in 100 tries continue # try again for the same job #--- generate rnd job #-- generate rnd job: ID CurrentUnitID = "%s-%d" % (UnitID, JobIndex % WLUnitMultiplicity) #-- generate rnd job: index UnitsDic['info'][JobIndex] = \ generateJobInfo( UnitDir, CurrentUnitID, JobIndex, Application, \ WLUnit, SubmitDurationMS, Submitter ) #-- generate rnd job: components UnitsDic['jobs'][JobIndex] = \ generateJobComponents( UnitDir, CurrentUnitID, JobIndex, WLUnit, PreComponentsList, JavaHomeDir, IbisAppsDir, IbisLibDir, JavaMaxMemoryLimitMB, Application, NCPUs ) #-- be more descriptive UnitsDic['info'][JobIndex]['description'] = \ UnitsDic['info'][JobIndex]['description'] + \ " Components: %d. NCPUs: %d." % (len(PreComponentsList), NCPUs) #--- done generating if UnitsDic['jobs'][JobIndex] == None: NJobs = NJobs - 1 #-- don't ack job, but make sure total no of jobs is decreased else: print "IBIS generator: Generated a job of type", Application, print "with", len(PreComponentsList), "components/", print NCPUs, "CPUs, in", UnitsDic['info'][JobIndex]['jdf'] JobIndex = JobIndex + 1 #-- ack job WLUnit['generatedjobs'] = JobIndex else: print "Complete workload" #--- generate all JobIndex = 0 RepetitionIndex = 0 while RepetitionIndex < WLUnitMultiplicity: for ComponentSize in ComponentSizeList: for NumComponents in NumComponentsList: CurrentUnitID = "%s-%d" % (UnitID, JobIndex % WLUnitMultiplicity) UnitsDic['info'][JobIndex] = \ generateJobInfo( UnitDir, CurrentUnitID, JobIndex, \ WLUnit, SubmitDurationMS, Submitter ) UnitsDic['jobs'][JobIndex] = \ generateJobComponents( UnitDir, CurrentUnitID, JobIndex, WLUnit, \ NumComponents, ComponentSize, Application, NCPUs ) JobIndex = JobIndex + 1 RepetitionIndex = RepetitionIndex + 1 if JobIndex > NJobs and NJobs > 0: print "Generated %d jobs and stopped because of the NJobs limit." break WLUnit['generatedjobs'] = JobIndex #raise Exception, "IBIS generator: Not implemented yet" return UnitsDic