def generateWorkloadUnit( UnitDir, UnitID, WLUnit, SubmitDurationMS, bGenerateRandom = 1 ): """ Out: UnitsDic o A dictionary with keys 'info' and 'jobs'. Notes: o UnitsDic['info'] contains a dictionary of jobs info, indexed with an integer counter o Each job info contains at least the keys name, description, jdf, submitCommand, runTime o UnitsDic['jobs'] contains a dictionary of jobs data, indexed with an integer counter o Each job data contains the list of components of that job o Each component in the list is a dictionary with at least the following fields: executable, stdout, stderr, name, description, directory, maxWallTime, arguments, env, stagein, stageout See also: utils/WLDocHandlers.WLSubmitJobKeys this_file.SMPI1Component.generateComponent """ global SMPI1_LastRunTime SMPI1Component.SMPI1_Exe = os.path.join( "/tmp", "smpi1t" ) SMPI1Component.SMPI1_ParKSizes = [ "0", "32", "128", "512", "1024" ] SMPI1Component.SMPI1_ParKSuperSizes = [ "0", "16", "32", "64", "128", "256" ] SMPI1Component.SMPI1_ParSupersteps = [ "1", "2", "5", "10", "20", "50", "100" ] SMPI1Component.SMPI1_ParMemoryKItems = [ "10", "25", "50", "100", "250", "500", "1000", "5000" ] SMPI1Component.SMPI1_ParMemoryElementsPerItem = [ "3", "4", "10", "100", "500", "1000" ] SMPI1Component.SMPI1_ParXChangeElementsPerStep = [ "100", "500", "1000", "10000", "50000" ] SMPI1Component.SMPI1_ParComputationPerMemoryItem = [ "2", "10", "100", "1000" ] SMPI1Component.SMPI1_RunTimeInMinutes = [ "2", "5", "10", "15" ] if 'otherinfo' in WLUnit.keys(): try: OneString = WLUnit['otherinfo']['Exe'] SMPI1Component.SMPI1_Exe = OneString except KeyError: pass try: OneIntList = AIParseUtils.readIntList( Text = WLUnit['otherinfo']['ParKSizes'], ItemSeparator = ',' ) SMPI1Component.SMPI1_ParKSizes = OneIntList except KeyError: pass try: OneIntList = AIParseUtils.readIntList( Text = WLUnit['otherinfo']['ParKSuperSizes'], ItemSeparator = ',' ) SMPI1Component.SMPI1_ParKSuperSizes = OneIntList except KeyError: pass try: OneIntList = AIParseUtils.readIntList( Text = WLUnit['otherinfo']['ParSupersteps'], ItemSeparator = ',' ) SMPI1Component.SMPI1_ParSupersteps = OneIntList except KeyError: pass try: OneIntList = AIParseUtils.readIntList( Text = WLUnit['otherinfo']['ParMemoryKItems'], ItemSeparator = ',' ) SMPI1Component.SMPI1_ParMemoryKItems = OneIntList except KeyError: pass try: OneIntList = AIParseUtils.readIntList( Text = WLUnit['otherinfo']['ParMemoryElementsPerItem'], ItemSeparator = ',' ) SMPI1Component.SMPI1_ParMemoryElementsPerItem = OneIntList except KeyError: pass try: OneIntList = AIParseUtils.readIntList( Text = WLUnit['otherinfo']['ParXChangeElementsPerStep'], ItemSeparator = ',' ) SMPI1Component.SMPI1_ParXChangeElementsPerStep = OneIntList except KeyError: pass try: OneIntList = AIParseUtils.readIntList( Text = WLUnit['otherinfo']['ParComputationPerMemoryItem'], ItemSeparator = ',' ) SMPI1Component.SMPI1_ParComputationPerMemoryItem = OneIntList except KeyError: pass try: OneIntList = AIParseUtils.readIntList( Text = WLUnit['otherinfo']['RunTimeInMinutes'], ItemSeparator = ',' ) SMPI1Component.SMPI1_RunTimeInMinutes = OneIntList except KeyError: pass try: NComponentsWithWeightsDic = \ AIParseUtils.readIntWithWeightsList( WLUnit['otherinfo']['NComponentsWithWeights'], DefaultWeight = 1.0 ) # NComponentsWithWeightsDic['Values']; NComponentsWithWeightsDic['TotalWeight'] except KeyError: raise Exception, "SMPI1 generator: Invalid NComponentsWithWeights info specified in the OtherInfo field." ##print ">>>>>> NComponentsWithWeightsDic", NComponentsWithWeightsDic[AIParseUtils.TOTAL_WEIGHT] try: TotalCPUsWithWeightsDic = \ AIParseUtils.readIntWithWeightsList( WLUnit['otherinfo']['TotalCPUsWithWeights'], DefaultWeight = 1.0 ) except KeyError: raise Exception, "SMPI1 generator: Invalid TotalCPUsWithWeights info specified in the OtherInfo field." ##print ">>>>>> TotalCPUsWithWeightsDic", TotalCPUsWithWeightsDic[AIParseUtils.TOTAL_WEIGHT] try: MinComponentSize = AIParseUtils.readInt(WLUnit['otherinfo']['MinComponentSize'], DefaultValue = 0) except KeyError: MinComponentSize = 1 #raise Exception, "SMPI1 generator: No MinComponentSize info specified in the OtherInfo field." try: MaxComponentSize = AIParseUtils.readInt(WLUnit['otherinfo']['MaxComponentSize'], DefaultValue = 0) except KeyError: MaxComponentSize = 10000000 #raise Exception, "SMPI1 generator: No MaxComponentSize info specified in the OtherInfo field." try: EqualCPUsPerComponent = AIParseUtils.readBoolean(WLUnit['otherinfo']['SMPI1AppsDir']) except KeyError: EqualCPUsPerComponent = 0 #raise Exception, "SMPI1 generator: No EqualCPUsPerComponent info specified in the OtherInfo field. Setting to default " + AIParseUtils.IntToBooleanDic[EqualCPUsPerComponent] + '.' try: AppBaseDir = WLUnit['otherinfo']['AppBaseDir'] except KeyError: AppBaseDir = '/tmp' #raise Exception, "SMPI1 generator: No AppBaseDir info specified in the OtherInfo field." print "WARNING! SMPI1 generator: No AppBaseDir info specified in the OtherInfo field.", "Assuming default:", AppBaseDir try: Application = WLUnit['otherinfo']['Application'] except KeyError: Application = 'smpi1t-gm' #raise Exception, "SMPI1 generator: No AppBaseDir info specified in the OtherInfo field." print "WARNING! SMPI1 generator: No Application info specified in the OtherInfo field.", "Assuming default:", Application try: Submitter = WLUnit['otherinfo']['Submitter'] except KeyError: Submitter = 'krunner -l DEBUG -g -e -o -f ${JDF}' print "WARNING! SMPI1 generator: No Submitter info specified in the OtherInfo field.", "Assuming default", Submitter try: SiteTypesWithWeightsDic = \ AIParseUtils.readStringWithWeightsList( WLUnit['otherinfo']['SiteTypesWithWeights'], DefaultWeight = 1.0 ) except KeyError: SiteTypesWithWeights = {AIParseUtils.VALUES:[], AIParseUtils.TOTAL_WEIGHT:0.0 } print "WARNING! SMPI1 generator: No SiteTypesWithWeights info specified in the OtherInfo field.\n\tSetting to default", SitesList ##print ">>>>>> SiteTypesWithWeightsDic", SiteTypesWithWeightsDic[AIParseUtils.TOTAL_WEIGHT] try: SitesWithWeightsDic = \ AIParseUtils.readStringWithWeightsList( WLUnit['otherinfo']['SitesWithWeights'], DefaultWeight = 1.0 ) except KeyError: SitesWithWeightsDic = {AIParseUtils.VALUES:[], AIParseUtils.TOTAL_WEIGHT:0.0 } print "WARNING! SMPI1 generator: No SitesWithWeights info specified in the OtherInfo field.\n\tSetting to default", SitesList ##print ">>>>>> SitesWithWeightsDic", SitesWithWeightsDic[AIParseUtils.TOTAL_WEIGHT] else: raise Exception, "SMPI1 generator: No OtherInfo data! Expected at least ComponentSize,NJobs,NComponents fields." UnitsDic = {} UnitsDic['info'] = {} UnitsDic['jobs'] = {} #-- init start time try: StartAt = AIParseUtils.readInt(WLUnit['otherinfo']['StartAt'], DefaultValue = 0) StartAt = StartAt * 1000 # the time is given is seconds except KeyError: StartAt = 0 SMPI1_LastRunTime = StartAt if 'FirstJobIndex' in WLUnit: JobIndex = int(WLUnit['FirstJobIndex']) else: JobIndex = 0 WLUnitMultiplicity = int(WLUnit['multiplicity']) if bGenerateRandom > 0 : index = 0 while index < WLUnitMultiplicity: # 1. generate a random site type (unordered or ordered) SiteType = AIRandomUtils.getRandomWeightedListElement( SiteTypesWithWeightsDic, ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT ) # 2. select the number of CPU for this job #TODO: repeat NCPUs selection until App can run on that many CPUs NCPUs = AIRandomUtils.getRandomWeightedListElement( TotalCPUsWithWeightsDic, ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT ) # 3. select the number of components and try to create subcomponents, # until successfully matching restrictions WhileCondition = 1 NTries = 100 while WhileCondition != 0 and NTries > 0: NTries = NTries - 1 NComponents = AIRandomUtils.getRandomWeightedListElement( NComponentsWithWeightsDic, ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT ) if NComponents > NCPUs: continue # impossible match, just retry NCPUsPerComponent = NCPUs / NComponents # ensure restrictions are met if NCPUsPerComponent < MinComponentSize or NCPUsPerComponent > MaxComponentSize: continue NRemainingComponents = NCPUs - NCPUsPerComponent * NComponents if NRemainingComponents > 0 and EqualCPUsPerComponent == 1: continue # >= -> including the extra CPU for some components if NRemainingComponents > 0 and NCPUsPerComponent == MaxComponentSize: continue PreComponentsList = [] for ComponentID in xrange(NComponents): # cpu count ToAssignHere = NCPUsPerComponent if ComponentID < NRemainingComponents: # assign the extra CPUs, round-robin ToAssignHere = ToAssignHere + 1 # site location if SiteType == 'ordered': Site = AIRandomUtils.getRandomWeightedListElement( SitesWithWeightsDic, ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT ) else: Site = '*' # create pre-component information PreComponent = {} PreComponent['id'] = ComponentID PreComponent['count'] = ToAssignHere PreComponent['location'] = Site # add pre-component PreComponentsList.append(PreComponent) WhileCondition = 0 if WhileCondition == 1: # failed to generate in 100 tries continue # try again for the same job #--- generate rnd job #-- generate job info UnitsDic['info'][index] = \ generateJobInfo( UnitDir, UnitID, JobIndex, \ WLUnit, SubmitDurationMS, Submitter ) #-- generate job components UnitsDic['jobs'][index] = \ generateJobComponents( UnitDir, UnitID, JobIndex, WLUnit, PreComponentsList, \ Application, AppBaseDir, \ bGenerateRandom ) index = index + 1 JobIndex = JobIndex + 1 WLUnit['generatedjobs'] = index else: index = 0 for Size in SMPI1Component.SMPI1_ParKSuperSizes: for N in SMPI1Component.SMPI1_ParSupersteps: for M in SMPI1Component.SMPI1_ParMemoryKItems: for S in SMPI1Component.SMPI1_ParMemoryElementsPerItem: for X in SMPI1Component.SMPI1_ParXChangeElementsPerStep: for C in SMPI1Component.SMPI1_ParComputationPerMemoryItem: raise Exception, "ERROR! SMPI1 is not finished!" #--- generate wall time once for all components of this job MaxWallTime = AIRandomUtils.getRandomListElement( SMPI1Component.SMPI1_RunTimeInSeconds ) #-- generate a random site type (unordered, ordered, ...) SiteType = AIRandomUtils.getRandomWeightedListElement( SiteTypesWithWeightsDic, ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT ) #-- generate job info UnitsDic['info'][index] = \ generateJobInfo( UnitDir, UnitID, JobIndex, \ WLUnit, SubmitDurationMS ) #-- generate job components UnitsDic['jobs'][index] = \ generateJobComponents( UnitDir, UnitID, JobIndex, WLUnit, \ bGenerateRandom, SiteType, Size, \ N, M, S, X, C, MaxWallTime ) index = index + 1 JobIndex = JobIndex + 1 #-- set generated jobs WLUnit['generatedjobs'] = index return UnitsDic
def generateWorkloadUnit( UnitDir, UnitID, WLUnit, SubmitDurationMS, bGenerateRandom = 1 ): """ Out: UnitsDic o A dictionary with keys 'info' and 'jobs'. Notes: o UnitsDic['info'] contains a dictionary of jobs info, indexed with an integer counter o Each job info contains at least the keys name, description, jdf, submitCommand, runTime o UnitsDic['jobs'] contains a dictionary of jobs data, indexed with an integer counter o Each job data contains the list of components of that job o Each component in the list is a dictionary with at least the following fields: executable, stdout, stderr, name, description, directory, maxWallTime, arguments, env, stagein, stageout See also: utils/WLDocHandlers.WLSubmitJobKeys this_file.SSERComponent.generateComponent """ global SSER_LastRunTime SSERComponent.SSER_Exe = "/home/antoniou/Synthetic_Apps/sseriot" #os.path.join( "/tmp", "ssert" ) SSERComponent.SSER_ParKSizes = [ "0", "32", "128", "512", "1024" ] SSERComponent.SSER_ParKSuperSizes = [ "0", "16", "32", "64", "128", "256" ] SSERComponent.SSER_ParSupersteps = [ "1", "2", "5", "10", "20", "50", "100" ] SSERComponent.SSER_ParMemoryKItems = [ "10", "25", "50", "100", "250", "500", "1000", "5000" ] SSERComponent.SSER_ParMemoryElementsPerItem = [ "3", "4", "10", "100", "500", "1000" ] SSERComponent.SSER_ParComputationPerMemoryItem = [ "2", "10", "100", "1000" ] SSERComponent.SSER_RunTimeInMinutes = [ "2", "5", "10", "15" ] if 'otherinfo' in WLUnit.keys(): try: OneString = WLUnit['otherinfo']['Exe'] SSERComponent.SSER_Exe = OneString except KeyError: pass try: OneIntList = AIParseUtils.readIntList( Text = WLUnit['otherinfo']['ParKSizes'], ItemSeparator = ',' ) SSERComponent.SSER_ParKSizes = OneIntList except KeyError: pass try: OneIntList = AIParseUtils.readIntList( Text = WLUnit['otherinfo']['ParKSuperSizes'], ItemSeparator = ',' ) SSERComponent.SSER_ParKSuperSizes = OneIntList except KeyError: pass try: OneIntList = AIParseUtils.readIntList( Text = WLUnit['otherinfo']['ParSupersteps'], ItemSeparator = ',' ) SSERComponent.SSER_ParSupersteps = OneIntList except KeyError: pass try: OneIntList = AIParseUtils.readIntList( Text = WLUnit['otherinfo']['ParMemoryKItems'], ItemSeparator = ',' ) SSERComponent.SSER_ParMemoryKItems = OneIntList except KeyError: pass try: OneIntList = AIParseUtils.readIntList( Text = WLUnit['otherinfo']['ParMemoryElementsPerItem'], ItemSeparator = ',' ) SSERComponent.SSER_ParMemoryElementsPerItem = OneIntList except KeyError: pass try: OneIntList = AIParseUtils.readIntList( Text = WLUnit['otherinfo']['ParComputationPerMemoryItem'], ItemSeparator = ',' ) SSERComponent.SSER_ParComputationPerMemoryItem = OneIntList except KeyError: pass try: OneIntList = AIParseUtils.readIntList( Text = WLUnit['otherinfo']['RunTimeInMinutes'], ItemSeparator = ',' ) SSERComponent.SSER_RunTimeInMinutes = OneIntList except KeyError: pass UnitsDic = {} UnitsDic['info'] = {} UnitsDic['jobs'] = {} #-- init start time try: StartAt = AIParseUtils.readInt(WLUnit['otherinfo']['StartAt'], DefaultValue = 0) StartAt = StartAt * 1000 # the time is given is seconds except KeyError: StartAt = 0 SSER_LastRunTime = StartAt if 'FirstJobIndex' in WLUnit: JobIndex = int(WLUnit['FirstJobIndex']) else: JobIndex = 0 if bGenerateRandom > 0 : index = 0 maxIndex = WLUnit['multiplicity'] while index < maxIndex: UnitsDic['info'][index] = \ generateJobInfo( UnitDir, UnitID, JobIndex, \ WLUnit, SubmitDurationMS ) UnitsDic['jobs'][index] = \ generateJobComponents( UnitDir, UnitID, JobIndex, WLUnit, \ bGenerateRandom ) index = index + 1 JobIndex = JobIndex + 1 WLUnit['generatedjobs'] = index else: index = 0 for Size in SSERComponent.SSER_ParKSuperSizes: for N in SSERComponent.SSER_ParSupersteps: for M in SSERComponent.SSER_ParMemoryKItems: for S in SSERComponent.SSER_ParMemoryElementsPerItem: for C in SSERComponent.SSER_ParComputationPerMemoryItem: for MaxWallTime in SSERComponent.SSER_RunTimeInSeconds: UnitsDic['info'][index] = \ generateJobInfo( UnitDir, UnitID, JobIndex, \ WLUnit, SubmitDurationMS ) UnitsDic['jobs'][index] = \ generateJobComponents( UnitDir, UnitID, JobIndex, WLUnit, \ bGenerateRandom, Size, N, M, S, C, MaxWallTime) index = index + 1 JobIndex = JobIndex + 1 #-- set generated jobs WLUnit['generatedjobs'] = index return UnitsDic
def generateWorkload(UnitDir, UnitID, WLUnit, SubmitDurationMS, bGenerateRandom=1): global HSH_LastRunTime # print WLUnit['otherinfo'] # {'RandomWorkload': 'true', 'NComponents': '1-8', 'NJobs': '50', 'ComponentSize':'4-8'} if bGenerateRandom > 0: print "Random workload" else: print "Full workload" if "otherinfo" in WLUnit.keys(): try: NComponents = WLUnit["otherinfo"]["NComponents"] except KeyError: raise Exception, "Hashim's generator: No NComponents info specified in the OtherInfo field." try: NumComponentsList = AIParseUtils.readIntRange(NComponents) if len(NumComponentsList) == 0: NumComponentsList = AIParseUtils.readIntList(NComponents) if len(NumComponentsList) == 0: raise Exception, "" except Exception, e: raise Exception, "Hashim's generator: Incorrect NComponents format!\n\tExpected NComponents=X-Y, with X,Y integers and X<Y or\n\tNComponents=V1;...;Vn, Vi integer. Got %s." % NComponents # print ">>>DEBUG>>> NumComponentsList", NumComponentsList try: NJobs = WLUnit["otherinfo"]["NJobs"] except KeyError: NJobs = "0" if bGenerateRandom > 0: # -- NJobs required if random, optional otherwise raise Exception, "Hashim's generator: No NJobs info specified in the OtherInfo field." try: NJobs = int(NJobs.strip()) except: raise Exception, "Hashim's generator: Incorrect NJobs format! Expected NJobs=X, with X integer." # print ">>>DEBUG>>> NJobs", NJobs try: ComponentSize = WLUnit["otherinfo"]["ComponentSize"] except KeyError: raise Exception, "Hashim's generator: No ComponentSize info specified in the OtherInfo field." try: ComponentSizeList = AIParseUtils.readIntRange(ComponentSize) if len(ComponentSizeList) == 0: ComponentSizeList = AIParseUtils.readIntList(ComponentSize) if len(ComponentSizeList) == 0: raise Exception, "" except: raise Exception, "Hashim's generator: Incorrect ComponentSize format!\n\tExpected ComponentSize=X-Y, with X,Y integers and X<Y or\n\tComponentSize=V1;...;Vn, Vi integer. Got %s." % ComponentSize # print ">>>DEBUG>>> ComponentSizeList", ComponentSizeList try: Submitter = WLUnit["otherinfo"]["Submitter"] except KeyError: Submitter = "drunner" print "WARNING! Hashim's generator: No Submitter info specified in the OtherInfo field.\n\tSetting to default", Submitter try: Application = WLUnit["otherinfo"]["Application"] except KeyError: Application = "Poisson-g2-gm" print "WARNING! Hashim's generator: No Application info specified in the OtherInfo field.\n\tSetting to default", Application