예제 #1
0
def generateJobComponents(UnitDir, UnitID, JobIndex, WLUnit, NumComponents, ComponentSize, Application):

    # minutes: 3, 15, 30, 700
    RunTimeInMinutesList = ["3", "15", "30", "700"]

    ComponentsList = []

    # --- generate wall time once for all components of this job
    MaxWallTime = AIRandomUtils.getRandomListElement(RunTimeInMinutesList)

    if UnitDir[0] != "/":
        if sys.platform.find("linux") >= 0:
            UnitDir = os.path.join(os.environ["PWD"], UnitDir)
        else:
            UnitDir = os.path.join(os.getcwd(), UnitDir)

    # -- hashim's workload definition
    ComponentDirName = "%d-%d-%d" % (JobIndex, NumComponents, ComponentSize)
    FullComponentDirName = os.path.join(UnitDir, "%s/" % UnitID, ComponentDirName)
    # --- Create output directory, if it does not exist
    if os.path.exists(FullComponentDirName):
        if not os.path.isdir(FullComponentDirName):
            print "Output for job", JobIndex, "(" + FullComponentDirName + ")", "exists, but is not a directory", "...skipping job"
            return -1
    else:
        try:
            os.makedirs(FullComponentDirName)
        except OSError, e:
            print "Cannot create output directory for job", JobIndex, "...skipping job"
            print "\tOS returned:", e
            return -1
예제 #2
0
def generateWorkload( OutDir, FullWLFileName, ListOfDefs, SubmitDurationMS,
                      AUnitGensLoader, AWLGensLoader, AJDFGeneratorsLoader, JDFGeneratorNames, 
                      DictionaryOfSites ,arrivalDistribution = None):
    print "STATUS! Starting workload generation!"
    
    # the number of workflow engines for which we generate output files
    NEngines = len(JDFGeneratorNames)
    print "STATUS! Generating %d workload formats \n" % NEngines
    FileFormatsArray = [] 
    
    #DagCommandsFormatsMap = {'condor_dag_wrapper.sh':'dagman', 'karajan_wrapper_remote.sh':'karajan'}
     
    #--- reset random generator
    AIRandomUtils.initRandom()
    
    #--- init arrival time generator
    try:
        ArrivalTimeRandom = AIRandomUtils.AIRandom( time() )
    except Exception, e:
        print "ERROR! Cannot instantiate the ArrivalTime random generator.", e
예제 #3
0
 def getNextValue(self):
     if self.SelectionMethod == IBISSelectionMethod.SM_Random:
         return self.RandomGenerator.getRandomListElement( self.DataList )
     elif self.SelectionMethod == IBISSelectionMethod.SM_WeightedRandom:
         return AIRandomUtils.getRandomWeightedListElement(
                    self.DataDic,
                    ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT 
                 )
     elif self.SelectionMethod == IBISSelectionMethod.SM_RoundRobin:
         if self.Counter >= self.MaxCounter: self.Counter = 0
         try:
             ReturnValue = self.DataDic[AIParseUtils.VALUES][self.Counter][0]
         except IndexError, e:
             print '>>>>>>>>>', e
             print '>>TRACE>>', traceback.print_exc()
             print '>>CNTR>>>', self.Counter, "/", self.MaxCounter
         self.Counter = self.Counter + 1
         return ReturnValue
예제 #4
0
def generateJobComponents( UnitDir, UnitID, JobIndex, WLUnit, \
    PreComponentsList, JavaHomeDir, IbisAppsDir, IbisLibDir, \
    JavaMaxMemoryLimitMB, Application, NCPUs):
    
    # minutes: 3, 15, 30, 700
    RunTimeInMinutesList = [ '3', '15', '30', '700' ]
    
    NumComponents = len(PreComponentsList)
    
    ComponentsList = []
    
    #--- generate wall time once for all components of this job
    MaxWallTime = AIRandomUtils.getRandomListElement( RunTimeInMinutesList )
    
    if UnitDir[0] != '/':
        if sys.platform.find("linux") >= 0:
            UnitDir = os.path.join( os.environ['PWD'], UnitDir )
        else:
            UnitDir = os.path.join( os.getcwd(), UnitDir )
        ### THE UGLY HACK: convert /disk1/home3/koala5/grenchmark to /home/koala5/grenchmark
        pos = UnitDir.find('/koala')
        if pos >= 0:
            UnitDir = '/home' + UnitDir[pos:]
            
    #-- IBIS workload definition
    ComponentDirName = "%d-%d" % ( JobIndex, NumComponents )
    FullComponentDirName = os.path.join( UnitDir, "%s/" % UnitID, ComponentDirName ) 
    #--- Create output directory, if it does not exist
    if os.path.exists( FullComponentDirName ):
        if not os.path.isdir( FullComponentDirName ):
            print "Output for job", JobIndex, "("+FullComponentDirName+")", "exists, but is not a directory", "...skipping job"
            return None
    else:
        try:
            os.makedirs( FullComponentDirName )
        except OSError, e:
            print "Cannot create output directory for job", JobIndex, "...skipping job"
            print '\tOS returned:', e
            return None
예제 #5
0
def ExpandApplication(Application):
    """ 
    replace an ambiguous application name (*) with a real name 
    (randomly selected from the global applications list) 
    """
    
    #-- find all matching names
    FullNamesList = []
    
    #-- replace the user friendly '*' with the Python RegExp correspondent
    Application = Application.replace('*', '\w+') # re's synthax requires + for 1/1+ matches
    SearchRE = re.compile(Application)
    #-- create the list of applications whose name match the request
    for App in IBIS_Apps.keys():
        if SearchRE.search('^' + App):
            FullNamesList.append(App)
            
    #-- select one name or die
    #print FullNamesList
    if len(FullNamesList) == 0:
        raise Exception, 'IBIS generator: Wrong application name ' + Application + ' (cannot expand).'
        
    FullApplicationName = AIRandomUtils.getRandomListElement(FullNamesList)
    return FullApplicationName
예제 #6
0
    def generateComponent( self, bGenerateRandom = 1, \
        Size = 0, N = 0, M = 0, S = 0, C = 0, MaxWallTime = 0 ):
        """
        
        Generates one component using SSER as the application. 
        
        This method does NOT write a physical JDF, but generates 
        all the needed parameters, directories, and input files 
        instead. The WLMain.generateWorkload is responsible for 
        actualy writing the JDFs. 
        
        In:
            bGenerateRandom -- whether this component is to be generated randomly 
                               (>0 for true, <=0 for false)
            size, N, M, S, C, MaxWallTime -- only valid if bGenerateRandom is 0
        
        Return:
            Int, >=0 on success, <0 otherwise
            
        Notes:
          o Upon success, the ComponentData will contain at least the following
            fields:
            executable    -- the complete path to an executable file
            stdout        -- a file that will receive the standard output messages
            stderr        -- a file that will receive the standard error messages
            name          -- a name for this component (if unique, the better)
            description   -- a description of this component
            directory     -- the directory where the component should run
            maxWallTime   -- the max time requested this app should run
            arguments     -- the list of arguments to be fed to the component's application
            env           -- the list of environmental variables, as (Name, Value) tuples
            stagein       -- a list of files to be staged in
            stageout      -- a list of files to be staged out
        
        """
        if bGenerateRandom > 0:
            if self.HaveInput > 0:
                InputSize = AIRandomUtils.getRandomListElement( SSERComponent.SSER_ParKSuperSizes )
            else:
                InputSize = 0
            if self.HaveOutput > 0:
                OutputSize = AIRandomUtils.getRandomListElement( SSERComponent.SSER_ParKSuperSizes )
            else:
                OutputSize = 0
            N = AIRandomUtils.getRandomListElement( SSERComponent.SSER_ParSupersteps )
            M = AIRandomUtils.getRandomListElement( SSERComponent.SSER_ParMemoryKItems )
            S = AIRandomUtils.getRandomListElement( SSERComponent.SSER_ParMemoryElementsPerItem )
            C = AIRandomUtils.getRandomListElement( SSERComponent.SSER_ParComputationPerMemoryItem )

#            N = SSERComponent.SSER_ParSupersteps[2]
#            M = AIRandomUtils.getRandomListElement( SSERComponent.SSER_ParMemoryKItems )
#            S = AIRandomUtils.getRandomListElement( SSERComponent.SSER_ParMemoryElementsPerItem )
#            C = SSERComponent.SSER_ParComputationPerMemoryItem[3]

            MaxWallTime = AIRandomUtils.getRandomListElement( SSERComponent.SSER_RunTimeInMinutes )
        
        if self.UnitDir[0] != '/':
            if sys.platform.find("linux") >= 0:
                self.UnitDir = os.path.join( os.environ['PWD'], self.UnitDir )
            else:
                self.UnitDir = os.path.join( os.getcwd(), self.UnitDir )
        
        ## too long component dir name
        #ComponentDirName = "%s_sser_%dx_%d_i%d_o%d" % \
        #        (self.ComponentData['id'], self.ComponentData['count'], int(N), int(InputSize), int(OutputSize))
        ComponentDirName = "%s_sser" % self.ComponentData['id']
        FullComponentDirName = os.path.join( self.UnitDir, ComponentDirName ) 
        #--- Create output directory, if it does not exist
        if os.path.exists( FullComponentDirName ):
            if not os.path.isdir( FullComponentDirName ):
                print "Output for job", self.ComponentData['id'], "("+FullComponentDirName+")", "exists, but is not a directory", "...skipping job"
                return -1
        else:
            try:
                os.makedirs( FullComponentDirName )
            except OSError, e:
                print "Cannot create output directory for job", self.ComponentData['id'] , "...skipping job"
                print '\tOS returned:', e
                return -1
예제 #7
0
        SSER_LastRunTime = RunTime
        InfoDic['runTime'] = "%.3f" % RunTime
        print "In sser-unit: RUNTIME IS ",InfoDic['runTime']
    except AIRandomUtils.AIRandomError, e:
        print "AIRandomUtils.AIRandomError", e
        InfoDic['runTime'] = None
    except Exception, e:
        print "ERROR!", e, "Name", Name, "Params", ParamsList
        InfoDic['runTime'] = None
        
    if (InfoDic['runTime'] == None) or \
       (float(InfoDic['runTime']) < 0) or (float(InfoDic['runTime']) > SubmitDurationMS):
        # all workload units are started within at most SubmitDurationMS miliseconds
        print "Wrong arrival time of", InfoDic['runTime'], "generated by", Name, ParamsList
        print "Resetting to default U(0,%d)" % SubmitDurationMS
        InfoDic['runTime'] = str(AIRandomUtils.getRandomInt(0, SubmitDurationMS))
    return InfoDic
    
def generateJobComponents( UnitDir, UnitID, JobIndex, WLUnit, bGenerateRandom = 1, \
    Size = 0, N = 20, M = None, S = None, C = 10, MaxWallTime = 0):
        
    ComponentsList = []
    nValidComponents = 0
    for ComponentID in WLUnit['components'].keys():
        
        Component = WLUnit['components'][ComponentID]
        WLCompName = "%s-%d-%s" % ( UnitID, JobIndex, ComponentID ) # added multiplicity id
        
        #-- set component
        ComponentData = {}
        if bGenerateRandom > 0:
예제 #8
0
def generateWorkload( UnitDir, UnitID, WLUnit, SubmitDurationMS, bGenerateRandom = 1 ):
    global SWF_LastRunTime
    
    SWFConstsInstance = SWFConsts()
    CurrentSWFParams = readParams( WLUnit )
    
    for Filter in CurrentSWFParams.SWF.Filters:
        print 'Filter', Filter, '->', CurrentSWFParams.SWF.Filters[Filter]
    
    JobsList, UNIXStartTime = readSWFFile( CurrentSWFParams.SWF.File, CurrentSWFParams )
    
    print "After filtering out,", len(JobsList), "jobs remained."
    ##for Job in JobsList:
    ##    print Job
    
    UnitsDic = {}
    UnitsDic['info'] = {}
    UnitsDic['jobs'] = {}
    
    #-- init start time
    try:
        StartAt = AIParseUtils.readInt(WLUnit['otherinfo']['StartAt'], DefaultValue = 0)
        StartAt = StartAt * 1000 # the time is given is seconds
    except KeyError:
        StartAt = 0
    SWF_LastRunTime = StartAt
    
    WLUnitMultiplicity = int(WLUnit['multiplicity'])
    
    FirstJob = None
    JobIndex = 0
    NJobsInList = len(JobsList)
    for GroupID in xrange(WLUnitMultiplicity):
        for JobID in xrange(NJobsInList):
            #JobIndex = JobID + GroupID * NJobsInList
            
            #--- get trace data
            Job = JobsList[JobID]
            ##print Job
            if FirstJob is None:
                FirstJob = Job
            
            # 2. select the number of CPU for this job
            #TODO: repeat NCPUs selection until App can run on that many CPUs
            NCPUs = Job[SWFConstsInstance.NCPUs]
            # 3. select the number of components and try to create subcomponents,
            #    until successfully matching restrictions
            WhileCondition = 1
            NTries = 250
            while WhileCondition != 0 and NTries > 0:
                
                NTries = NTries - 1
                
                NComponents = AIRandomUtils.getRandomWeightedListElement(
                                    CurrentSWFParams.NComponentsWithWeightsDic,
                                    ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT 
                                    )
                                    
                if NComponents > NCPUs: 
                    continue # impossible match, just retry
                
                NCPUsPerComponent = NCPUs / NComponents
                # ensure restrictions are met 
                if NCPUsPerComponent < CurrentSWFParams.MinComponentSize or \
                   NCPUsPerComponent > CurrentSWFParams.MaxComponentSize:
                    continue
                
                NRemainingComponents = NCPUs - NCPUsPerComponent * NComponents
                if NRemainingComponents > 0 and CurrentSWFParams.EqualCPUsPerComponent == 1:
                    continue
                    
                # >= -> including the extra CPU for some components
                if NRemainingComponents > 0 and NCPUsPerComponent == CurrentSWFParams.MaxComponentSize:
                    continue
                
                SiteType = AIRandomUtils.getRandomWeightedListElement(
                                    CurrentSWFParams.SiteTypesWithWeightsDic,
                                    ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT 
                                    )
                
                PreComponentsList = []
                for ComponentID in xrange(NComponents):
                    
                    # cpu count
                    ToAssignHere = NCPUsPerComponent
                    if ComponentID < NRemainingComponents: # assign the extra CPUs, round-robin
                        ToAssignHere = ToAssignHere + 1
                        
                    # site location
                    if SiteType == 'ordered':
                        Site = AIRandomUtils.getRandomWeightedListElement(
                                    CurrentSWFParams.SitesWithWeightsDic,
                                    ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT 
                                    )
                    else:
                        Site = '*'
                        
                    # create pre-component information
                    PreComponent = {}
                    PreComponent['id'] = ComponentID
                    PreComponent['count'] = ToAssignHere
                    PreComponent['location'] = Site
                    
                    # add pre-component
                    PreComponentsList.append(PreComponent)
                    
                WhileCondition = 0
                
            # skip errors
            if WhileCondition != 0:
                continue
                    
            #--- generate job
            Application = CurrentSWFParams.Application
            Submitter = CurrentSWFParams.Submitter
            ApplicationBinaryFullPath = os.path.join(CurrentSWFParams.AppBaseDir, CurrentSWFParams.Application)
            SWFConstsInstace = SWFConsts()
            # convert the start time from ms to s
            JobStartTime = 1000 * (Job[SWFConstsInstace.SubmitTime] - FirstJob[SWFConstsInstace.SubmitTime])
            if JobStartTime < 0:
                print "Flurry! JobStartTime < 0"
                print "FirstJob=", FirstJob
                print "Job=", Job
            
            #-- generate job: ID
            CurrentUnitID = "%s-%d-%d" % (UnitID, GroupID, JobIndex % NJobsInList)
            #-- generate job: index
            UnitsDic['info'][JobIndex] = \
                        generateJobInfo( UnitDir, CurrentUnitID, JobIndex, Application, \
                                         WLUnit, SubmitDurationMS, Submitter, \
                                         CurrentSWFParams.UseArrivalTimeDistribution, JobStartTime )
            #-- generate job: components
            UnitsDic['jobs'][JobIndex] = generateJobComponents( 
                            UnitDir, CurrentUnitID, JobIndex, WLUnit, 
                            PreComponentsList, ApplicationBinaryFullPath, \
                            NCPUs, Job, CurrentSWFParams.MaxWallTime )
                            
            #-- be more descriptive
            UnitsDic['info'][JobIndex]['description'] = \
                UnitsDic['info'][JobIndex]['description'] + \
                " Components: %d. NCPUs: %d." % (len(PreComponentsList), NCPUs) 
                            
            #--- done generating
            if UnitsDic['jobs'][JobIndex] == None:
                NJobs = NJobs - 1 #-- don't ack job, but make sure total no of jobs is decreased
            else:
                print "SWF generator: Generated a job of type", Application, 
                print "with", len(PreComponentsList), "components/", 
                print NCPUs, "CPUs, in", UnitsDic['info'][JobIndex]['jdf']
                JobIndex = JobIndex + 1
              
        # add the delay between separate units
        SWF_LastRunTime = SWF_LastRunTime + CurrentSWFParams.DelayBetweenUnits
        
    print "Generated", JobIndex, "jobs."
    WLUnit['generatedjobs'] = JobIndex

    #raise Exception, "SWF generator: Not implemented yet"
    return UnitsDic
예제 #9
0
def generateJDF(OutFileName, ListOfComponents):
    """ 
    Generate a OGF JSDL File for a given workload unit. 
    
    In:
        OutFileName      -- the target file for writing the JDF of this workload unit
        ListOfComponents -- a list of components
                            Each component is a dictionary and has defined at 
                            least the following keys:
                            id, executable, count, description, directory, 
                            maxWallTime, arguments, env, stagein, stageout,
                            stdout, stderr
    """

    print "STATUS! Write JDF file", OutFileName,
    OutFile = open(OutFileName, "w")

    index = 0
    nComponents = len(ListOfComponents)
    Component = ListOfComponents[0]  # one component

    writeJsdlHeader(OutFile)
    OutFile.write("<jsdl:JobDescription>\n")

    """
    <jsdl:Application>
      <jsdl:ApplicationName>ls</jsdl:ApplicationName>
      <jsdl-posix:POSIXApplication>
        <jsdl-posix:Executable>/bin/ls</jsdl-posix:Executable>
        <jsdl-posix:Argument>-la file.txt</jsdl-posix:Argument>
        <jsdl-posix:Environment name="LD_LIBRARY_PATH">/usr/local/lib</jsdl-posix:Environment>
        <jsdl-posix:Input>/dev/null</jsdl-posix:Input>
        <jsdl-posix:Output>stdout.${JOB_ID}</jsdl-posix:Output>
        <jsdl-posix:Error>stderr.${JOB_ID}</jsdl-posix:Error>
      </jsdl-posix:POSIXApplication>
    </jsdl:Application>
    """
    OutFile.write("<jsdl:Application>\n")
    OutFile.write("<jsdl:ApplicationName>")
    OutFile.write("sser" + str(AIRandomUtils.getRandomInt(0, int(2 ** 64))))
    OutFile.write("</jsdl:ApplicationName>\n")

    OutFile.write("<jsdl-posix:POSIXApplication>\n")
    OutFile.write("<jsdl-posix:Executable>")
    OutFile.write(str(Component["executable"]))
    OutFile.write("</jsdl-posix:Executable>\n")

    args = Component["arguments"]
    writeApplicationArguments(OutFile, args)
    stdout = Component["stdout"]
    OutFile.write("<jsdl-posix:Output>" + stdout + "</jsdl-posix:Output>\n")
    stderr = Component["stderr"]
    OutFile.write("<jsdl-posix:Error>" + stderr + "</jsdl-posix:Error>\n")
    # OutFile.write('<jsdl-posix:Input></jsdl-posix:Input>\n')

    writeApplicationEnvVariables(OutFile, Component["env"])

    OutFile.write('<jsdl-posix:Environment name="LD_LIBRARY_PATH">/usr/local/lib</jsdl-posix:Environment>')

    OutFile.write("\n</jsdl-posix:POSIXApplication>\n")

    OutFile.write("</jsdl:Application>\n")

    """
        <jsdl:DataStaging>
      <jsdl:FileName>file.txt</jsdl:FileName>
      <jsdl:CreationFlag>overwrite</jsdl:CreationFlag>
      <jsdl:DeleteOnTermination>true</jsdl:DeleteOnTermination>
      <jsdl:Source>
        <jsdl:URI>gsiftp://hydrus.dacya.ucm.es/home/jose/file1.txt</jsdl:URI>
      </jsdl:Source>
    </jsdl:DataStaging>
    """
    stagein = Component["stagein"]
    stageout = Component["stageout"]
    writeStageInFiles(OutFile, stagein)
    writeStageOutFiles(OutFile, stageout)

    OutFile.write("</jsdl:JobDescription>")
    writeJsdlFooter(OutFile)
    OutFile.close()
    print "...done"
예제 #10
0
def runJob(data):
    # 1. check if the job can be run (deps & time)
    # 1.1. yes: run the job
    # 1.1.1.    if job run successfully
    # 1.1.1.1.  yes: mark job as correct
    # 1.1.1.2.  no: put the job back into the pool, if not failed (too many failures)
    # 1.2. no: put the job back into the pool

    # -- compute due sleep time - modified (corina): sleep time will always be 0
    # sleepDuration = max( 0, data['firstSubmission'] + data['startTime'] - time.time())
    sleepDuration = 0

    # 1. check if the job can be run (time)
    if sleepDuration <= 0:  # job is due to run
        # -- start timers
        result = {}
        result["start"] = time.time()
        result["wakeup"] = time.time()
        result["submit"] = time.time()
        # 1. check if the job can be run (time)
        JobID = data["id"]
        # print "##### [runJob] job is due to run: %s " % JobID

        CurrentCompositeApplicationData = data[".CompositeApplicationData"]
        if CurrentCompositeApplicationData.isRunnable(JobID):  # job can run
            # 1.1. yes: run the job
            patchSleepDuration = AIRandomUtils.getRandomInt(0, 10)
            time.sleep(patchSleepDuration)
            Command = data["commandLine"]
            outputString = "(" + data["id"] + ") runs command:\n    " + Command
            print outputString
            if data["NoSubmit"] != 1:
                try:
                    cmdOutput = AISystemUtil.getCommandOutput2(Command)
                    result["status"] = "SUCCEEDED"
                    CurrentCompositeApplicationData.triggerJobFinished(JobID)

                except RuntimeError, e:
                    print "Job", JobID, "returned:", e
                    cmdOutput = ""
                    # job failed ?
                    CurrentCompositeApplicationData.triggerJobFailure(JobID)
                    if CurrentCompositeApplicationData.isFailed(JobID):
                        result["status"] = "FAILED"
                        CurrentCompositeApplicationData.propagateFailure(JobID)
                        print "Job", JobID, "was declared failed (too many failures)"

                    else:
                        result["status"] = "RETRY"
                        print "Job", JobID, "failed... rescheduling"
                        return None

                outputString = "Output (" + JobID + "): '" + cmdOutput + "'"
                print outputString
            else:  # no submit -> just mark as succeessful
                result["status"] = "SUCCEEDED"

            result["exit"] = time.time()
            # -- return all the times
            ##print "Job", JobID, "Send good result", result['status']
            return result

        elif CurrentCompositeApplicationData.isFailed(JobID):
            result["exit"] = time.time()
            result["status"] = "FAILED"
            # -- return all the times
            ##print "Job", JobID, "Send good result", result['status']
            return result
예제 #11
0
def getWLName( Prefix = "WL" ):
    return "%s-%s-%3.3d" % \
        ( Prefix, strftime('%Y-%m-%d_%H-%M', gmtime(time()) ), 
          AIRandomUtils.getRandomInt(0,999) )
예제 #12
0
                    #--- create subunits, one per composite structure
                    SubUnit = {}
                    SubUnit['.CompositeStructureIndex'] = CompositeStructureIndex
                    SubUnit['.CompositeStructureID'] = CompositeStructure.getID()
                    SubUnit['.NJobsToGenerate'] = CompositeStructure.getNNodes()
                    SubUnit['id'] = "%s" % CompositeStructure.getID()
                    SubUnit['IsWorkload'] = 0
                    SubUnit['composite'] = 0
                    SubUnit['multiplicity'] = 1 
                    
                    # NOTE: all workload units need to use this information to generate correctly
                    #       their JDF names...
                    #SubUnit['FirstJobIndex'] = JobIndex
                    
                    CurrentAppType = AIRandomUtils.getRandomWeightedListElement(
                                        AppTypesWithWeights,
                                        ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT 
                                        )
                    SubUnit['apptype'] = CurrentAppType
                    #SubUnit['arrivaltimeinfo'] = ('Zero', []) # all jobs in the same composite application arrive at the same time, 0
                    SubUnit['arrivaltimeinfo'] = WLUnit['arrivaltimeinfo'] 
                    SubUnit['arrivaltimefunc'] = WLUnit['arrivaltimefunc']

                    
                    SubUnit['otherinfo'] = {}
                    for item in WLUnit['otherinfo']:
                        CurrentAppStamp = CurrentAppType + '.'
                        if item.find(CurrentAppStamp) == 0: # this other info item pertains the current application type
                            EntryInternalName = item[len(CurrentAppStamp):]
                            SubUnit['otherinfo'][EntryInternalName] = WLUnit['otherinfo'][item]
                        
                    # TODO: composite apps site info quick-hack fix !!!
예제 #13
0
def generateWorkloadUnit( UnitDir, UnitID, WLUnit, SubmitDurationMS, bGenerateRandom = 1 ):
    """ 
    Out:
        UnitsDic 
            o A dictionary with keys 'info' and 'jobs'.
            
    Notes:
        o UnitsDic['info'] contains a dictionary of jobs info, 
          indexed with an integer counter
        o Each job info contains at least the keys
          name, description, jdf, submitCommand, runTime
        o UnitsDic['jobs'] contains a dictionary of jobs data, 
          indexed with an integer counter
        o Each job data contains the list of components of that job
        o Each component in the list is a dictionary with at least the 
          following fields: executable, stdout, stderr, name, description,
          directory, maxWallTime, arguments, env, stagein, stageout 
        
    See also:
        utils/WLDocHandlers.WLSubmitJobKeys
        this_file.SMPI1Component.generateComponent
    """
    
    global SMPI1_LastRunTime
    
    SMPI1Component.SMPI1_Exe = os.path.join( "/tmp", "smpi1t" )
    SMPI1Component.SMPI1_ParKSizes = [ "0", "32", "128", "512", "1024" ]
    SMPI1Component.SMPI1_ParKSuperSizes = [ "0", "16", "32", "64", "128", "256" ]
    SMPI1Component.SMPI1_ParSupersteps = [ "1", "2", "5", "10", "20", "50", "100" ]
    SMPI1Component.SMPI1_ParMemoryKItems = [ "10", "25", "50", "100", "250", "500", "1000", "5000" ]
    SMPI1Component.SMPI1_ParMemoryElementsPerItem = [ "3", "4", "10", "100", "500", "1000" ]
    SMPI1Component.SMPI1_ParXChangeElementsPerStep = [ "100", "500", "1000", "10000", "50000" ]
    SMPI1Component.SMPI1_ParComputationPerMemoryItem = [ "2", "10", "100", "1000" ]        
    SMPI1Component.SMPI1_RunTimeInMinutes = [ "2", "5", "10", "15" ]
    
    if 'otherinfo' in WLUnit.keys():
        
        try:
            OneString = WLUnit['otherinfo']['Exe']
            SMPI1Component.SMPI1_Exe = OneString
        except KeyError:
            pass
        
        try:
            OneIntList = AIParseUtils.readIntList( 
                    Text = WLUnit['otherinfo']['ParKSizes'], 
                    ItemSeparator = ',' )
            SMPI1Component.SMPI1_ParKSizes = OneIntList
        except KeyError:
            pass
            
        try:
            OneIntList = AIParseUtils.readIntList( 
                    Text = WLUnit['otherinfo']['ParKSuperSizes'], 
                    ItemSeparator = ',' )
            SMPI1Component.SMPI1_ParKSuperSizes = OneIntList
        except KeyError:
            pass
            
        try:
            OneIntList = AIParseUtils.readIntList( 
                    Text = WLUnit['otherinfo']['ParSupersteps'], 
                    ItemSeparator = ',' )
            SMPI1Component.SMPI1_ParSupersteps = OneIntList
        except KeyError:
            pass
            
        try:
            OneIntList = AIParseUtils.readIntList( 
                    Text = WLUnit['otherinfo']['ParMemoryKItems'], 
                    ItemSeparator = ',' )
            SMPI1Component.SMPI1_ParMemoryKItems = OneIntList
        except KeyError:
            pass
            
        try:
            OneIntList = AIParseUtils.readIntList( 
                    Text = WLUnit['otherinfo']['ParMemoryElementsPerItem'], 
                    ItemSeparator = ',' )
            SMPI1Component.SMPI1_ParMemoryElementsPerItem = OneIntList
        except KeyError:
            pass
            
        try:
            OneIntList = AIParseUtils.readIntList( 
                    Text = WLUnit['otherinfo']['ParXChangeElementsPerStep'], 
                    ItemSeparator = ',' )
            SMPI1Component.SMPI1_ParXChangeElementsPerStep = OneIntList
        except KeyError:
            pass
            
        try:
            OneIntList = AIParseUtils.readIntList( 
                    Text = WLUnit['otherinfo']['ParComputationPerMemoryItem'], 
                    ItemSeparator = ',' )
            SMPI1Component.SMPI1_ParComputationPerMemoryItem = OneIntList
        except KeyError:
            pass
            
        try:
            OneIntList = AIParseUtils.readIntList( 
                    Text = WLUnit['otherinfo']['RunTimeInMinutes'], 
                    ItemSeparator = ',' )
            SMPI1Component.SMPI1_RunTimeInMinutes = OneIntList
        except KeyError:
            pass
        
        try:
            NComponentsWithWeightsDic = \
                AIParseUtils.readIntWithWeightsList( 
                    WLUnit['otherinfo']['NComponentsWithWeights'], 
                    DefaultWeight = 1.0 
                    )
            # NComponentsWithWeightsDic['Values']; NComponentsWithWeightsDic['TotalWeight']
        except KeyError:
            raise Exception, "SMPI1 generator: Invalid NComponentsWithWeights info specified in the OtherInfo field."
        ##print ">>>>>> NComponentsWithWeightsDic", NComponentsWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
        
        try:
            TotalCPUsWithWeightsDic = \
                AIParseUtils.readIntWithWeightsList( 
                    WLUnit['otherinfo']['TotalCPUsWithWeights'], 
                    DefaultWeight = 1.0 
                    )
        except KeyError:
            raise Exception, "SMPI1 generator: Invalid TotalCPUsWithWeights info specified in the OtherInfo field."
        ##print ">>>>>> TotalCPUsWithWeightsDic", TotalCPUsWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
        
        
        try:
            MinComponentSize = AIParseUtils.readInt(WLUnit['otherinfo']['MinComponentSize'], DefaultValue = 0)
        except KeyError:
            MinComponentSize = 1
            #raise Exception, "SMPI1 generator: No MinComponentSize info specified in the OtherInfo field."
            
        try:
            MaxComponentSize = AIParseUtils.readInt(WLUnit['otherinfo']['MaxComponentSize'], DefaultValue = 0)
        except KeyError:
            MaxComponentSize = 10000000
            #raise Exception, "SMPI1 generator: No MaxComponentSize info specified in the OtherInfo field."
        
        try:
            EqualCPUsPerComponent = AIParseUtils.readBoolean(WLUnit['otherinfo']['SMPI1AppsDir'])
        except KeyError:
            EqualCPUsPerComponent = 0
            #raise Exception, "SMPI1 generator: No EqualCPUsPerComponent info specified in the OtherInfo field. Setting to default " + AIParseUtils.IntToBooleanDic[EqualCPUsPerComponent] + '.'
        
        try:
            AppBaseDir = WLUnit['otherinfo']['AppBaseDir']
        except KeyError:
            AppBaseDir = '/tmp'
            #raise Exception, "SMPI1 generator: No AppBaseDir info specified in the OtherInfo field."
            print "WARNING! SMPI1 generator: No AppBaseDir info specified in the OtherInfo field.", "Assuming default:", AppBaseDir
            
        try:
            Application = WLUnit['otherinfo']['Application']
        except KeyError:
            Application = 'smpi1t-gm'
            #raise Exception, "SMPI1 generator: No AppBaseDir info specified in the OtherInfo field."
            print "WARNING! SMPI1 generator: No Application info specified in the OtherInfo field.", "Assuming default:", Application
        
        try:
            Submitter = WLUnit['otherinfo']['Submitter']
        except KeyError:
            Submitter = 'krunner -l DEBUG -g -e -o -f ${JDF}'
            print "WARNING! SMPI1 generator: No Submitter info specified in the OtherInfo field.", "Assuming default", Submitter
            
        try:
            SiteTypesWithWeightsDic = \
                AIParseUtils.readStringWithWeightsList( 
                    WLUnit['otherinfo']['SiteTypesWithWeights'], 
                    DefaultWeight = 1.0 
                    )
        except KeyError:
            SiteTypesWithWeights = {AIParseUtils.VALUES:[], AIParseUtils.TOTAL_WEIGHT:0.0 }
            print "WARNING! SMPI1 generator: No SiteTypesWithWeights info specified in the OtherInfo field.\n\tSetting to default", SitesList
        ##print ">>>>>> SiteTypesWithWeightsDic", SiteTypesWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
        
        try:
            SitesWithWeightsDic = \
                AIParseUtils.readStringWithWeightsList( 
                    WLUnit['otherinfo']['SitesWithWeights'], 
                    DefaultWeight = 1.0 
                    )
        except KeyError:
            SitesWithWeightsDic = {AIParseUtils.VALUES:[], AIParseUtils.TOTAL_WEIGHT:0.0 }
            print "WARNING! SMPI1 generator: No SitesWithWeights info specified in the OtherInfo field.\n\tSetting to default", SitesList
        ##print ">>>>>> SitesWithWeightsDic", SitesWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
            
    else:
        raise Exception, "SMPI1 generator: No OtherInfo data! Expected at least ComponentSize,NJobs,NComponents fields."
    
    
    UnitsDic = {}
    UnitsDic['info'] = {}
    UnitsDic['jobs'] = {}
    
    #-- init start time
    try:
        StartAt = AIParseUtils.readInt(WLUnit['otherinfo']['StartAt'], DefaultValue = 0)
        StartAt = StartAt * 1000 # the time is given is seconds
    except KeyError:
        StartAt = 0
    SMPI1_LastRunTime = StartAt
    
    if 'FirstJobIndex' in WLUnit:
        JobIndex = int(WLUnit['FirstJobIndex'])
    else: 
        JobIndex = 0
    
    WLUnitMultiplicity = int(WLUnit['multiplicity'])
    if bGenerateRandom > 0 :
        
        index = 0
        while index < WLUnitMultiplicity:
            # 1. generate a random site type (unordered or ordered)
            SiteType = AIRandomUtils.getRandomWeightedListElement(
                            SiteTypesWithWeightsDic,
                            ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT 
                            )
                            
            # 2. select the number of CPU for this job
            #TODO: repeat NCPUs selection until App can run on that many CPUs
            NCPUs = AIRandomUtils.getRandomWeightedListElement(
                       TotalCPUsWithWeightsDic,
                       ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT 
                    )
            # 3. select the number of components and try to create subcomponents,
            #    until successfully matching restrictions
            WhileCondition = 1
            NTries = 100
            while WhileCondition != 0 and NTries > 0:
                
                NTries = NTries - 1
                
                NComponents = AIRandomUtils.getRandomWeightedListElement(
                                    NComponentsWithWeightsDic,
                                    ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT 
                                    )
                                    
                if NComponents > NCPUs: 
                    continue # impossible match, just retry
                
                NCPUsPerComponent = NCPUs / NComponents
                # ensure restrictions are met
                if NCPUsPerComponent < MinComponentSize or NCPUsPerComponent > MaxComponentSize:
                    continue
                
                NRemainingComponents = NCPUs - NCPUsPerComponent * NComponents
                if NRemainingComponents > 0 and EqualCPUsPerComponent == 1:
                    continue
                
                # >= -> including the extra CPU for some components
                if NRemainingComponents > 0 and NCPUsPerComponent == MaxComponentSize:
                    continue
                
                PreComponentsList = []
                for ComponentID in xrange(NComponents):
                    
                    # cpu count
                    ToAssignHere = NCPUsPerComponent
                    if ComponentID < NRemainingComponents: # assign the extra CPUs, round-robin
                        ToAssignHere = ToAssignHere + 1
                        
                    # site location
                    if SiteType == 'ordered':
                        Site = AIRandomUtils.getRandomWeightedListElement(
                                    SitesWithWeightsDic,
                                    ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT 
                                    )
                    else:
                        Site = '*'
                        
                    # create pre-component information
                    PreComponent = {}
                    PreComponent['id'] = ComponentID
                    PreComponent['count'] = ToAssignHere
                    PreComponent['location'] = Site
                    
                    # add pre-component
                    PreComponentsList.append(PreComponent)
                    
                WhileCondition = 0
                    
            if WhileCondition == 1: # failed to generate in 100 tries
                continue # try again for the same job
                
            #--- generate rnd job
            #-- generate job info
            UnitsDic['info'][index] = \
               generateJobInfo( UnitDir, UnitID, JobIndex, \
                                WLUnit, SubmitDurationMS, Submitter )
                                
            #-- generate job components
            UnitsDic['jobs'][index] = \
               generateJobComponents( UnitDir, UnitID, JobIndex, WLUnit, PreComponentsList, \
                                      Application, AppBaseDir, \
                                      bGenerateRandom )
            index = index + 1
            JobIndex = JobIndex + 1
        WLUnit['generatedjobs'] = index
        
    else:
        index = 0
        for Size in SMPI1Component.SMPI1_ParKSuperSizes:
            for N in SMPI1Component.SMPI1_ParSupersteps:
                for M in SMPI1Component.SMPI1_ParMemoryKItems:
                    for S in SMPI1Component.SMPI1_ParMemoryElementsPerItem:
                        for X in SMPI1Component.SMPI1_ParXChangeElementsPerStep:
                            for C in SMPI1Component.SMPI1_ParComputationPerMemoryItem:
                                
                                raise Exception, "ERROR! SMPI1 is not finished!"
                                
                                #--- generate wall time once for all components of this job
                                MaxWallTime = AIRandomUtils.getRandomListElement( SMPI1Component.SMPI1_RunTimeInSeconds )
                                #-- generate a random site type (unordered, ordered, ...)
                                SiteType = AIRandomUtils.getRandomWeightedListElement(
                                                SiteTypesWithWeightsDic,
                                                ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT 
                                                )
                                #-- generate job info
                                UnitsDic['info'][index] = \
                                   generateJobInfo( UnitDir, UnitID, JobIndex, \
                                                    WLUnit, SubmitDurationMS )
                                #-- generate job components
                                UnitsDic['jobs'][index] = \
                                   generateJobComponents( UnitDir, UnitID, JobIndex, WLUnit, \
                                                          bGenerateRandom, SiteType, Size, \
                                                          N, M, S, X, C, MaxWallTime )
                                index = index + 1
                                JobIndex = JobIndex + 1
        #-- set generated jobs
        WLUnit['generatedjobs'] = index
        
    return UnitsDic
예제 #14
0
def generateWorkload( UnitDir, UnitID, WLUnit, SubmitDurationMS, bGenerateRandom = 1 ):
    global IBIS_LastRunTime
    #print WLUnit['otherinfo']
    #{'RandomWorkload': 'true', 'NComponents': '1-8', 'NJobs': '50', 'ComponentSize':'4-8'}
    
    if bGenerateRandom > 0:
        print "Random workload"
    else:
        print "Full workload"
    
    if 'otherinfo' in WLUnit.keys():
        
        try:
            ##print ">>>>>> NJobs" 
            NJobs = AIParseUtils.readInt(WLUnit['otherinfo']['NJobs'], DefaultValue = 0)
        except KeyError:
            NJobs = 0
            if bGenerateRandom > 0: #-- NJobs required if random, optional otherwise
                raise Exception, "IBIS generator: No NJobs info specified in the OtherInfo field."
        
        try:
            NComponentsWithWeightsDic = \
                AIParseUtils.readIntWithWeightsList( 
                    WLUnit['otherinfo']['NComponentsWithWeights'], 
                    DefaultWeight = 1.0 
                    )
            # NComponentsWithWeightsDic['Values']; NComponentsWithWeightsDic['TotalWeight']
        except KeyError:
            raise Exception, "IBIS generator: Invalid NComponentsWithWeights info specified in the OtherInfo field."
        ##print ">>>>>> NComponentsWithWeightsDic", NComponentsWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
        
        try:
            TotalCPUsWithWeightsDic = \
                AIParseUtils.readIntWithWeightsList( 
                    WLUnit['otherinfo']['TotalCPUsWithWeights'], 
                    DefaultWeight = 1.0 
                    )
        except KeyError:
            raise Exception, "IBIS generator: Invalid TotalCPUsWithWeights info specified in the OtherInfo field."
        ##print ">>>>>> TotalCPUsWithWeightsDic", TotalCPUsWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
        
        
        try:
            MinComponentSize = AIParseUtils.readInt(WLUnit['otherinfo']['MinComponentSize'], DefaultValue = 0)
        except KeyError:
            MinComponentSize = 1
            #raise Exception, "IBIS generator: No MinComponentSize info specified in the OtherInfo field."
            
        try:
            MaxComponentSize = AIParseUtils.readInt(WLUnit['otherinfo']['MaxComponentSize'], DefaultValue = 0)
        except KeyError:
            MaxComponentSize = 10000000
            #raise Exception, "IBIS generator: No MaxComponentSize info specified in the OtherInfo field."
        
        try:
            EqualCPUsPerComponent = AIParseUtils.readBoolean(WLUnit['otherinfo']['IbisAppsDir'])
        except KeyError:
            EqualCPUsPerComponent = 0
            #raise Exception, "IBIS generator: No EqualCPUsPerComponent info specified in the OtherInfo field. Setting to default " + AIParseUtils.IntToBooleanDic[EqualCPUsPerComponent] + '.'
        
        try:
            IbisAppsDir = WLUnit['otherinfo']['IbisAppsDir']
        except KeyError:
            raise Exception, "IBIS generator: No IbisAppsDir info specified in the OtherInfo field."
            
        try:
            IbisLibDir = WLUnit['otherinfo']['IbisLibDir']
        except KeyError:
            raise Exception, "IBIS generator: No IbisLibDir info specified in the OtherInfo field."
            
        try:
            JavaHomeDir = WLUnit['otherinfo']['JavaHomeDir']
        except KeyError:
            raise Exception, "IBIS generator: No JavaHomeDir info specified in the OtherInfo field."
        
        try:
            AppsWithWeightsDic = \
                AIParseUtils.readStringWithWeightsList( 
                    WLUnit['otherinfo']['AppsWithWeights'], 
                    DefaultWeight = 1.0 
                    )
        except KeyError:
            raise Exception, "IBIS generator: Invalid AppsWithWeights info specified in the OtherInfo field."
        ###print ">>>>>> AppsWithWeightsDic", AppsWithWeightsDic#[AIParseUtils.TOTAL_WEIGHT]
        
        try:
            Submitter = WLUnit['otherinfo']['Submitter']
        except KeyError:
            Submitter = 'grunner'
            print "WARNING! IBIS generator: No Submitter info specified in the OtherInfo field.\n\tSetting to default", Submitter
            
        try:
            SiteTypesWithWeightsDic = \
                AIParseUtils.readStringWithWeightsList( 
                    WLUnit['otherinfo']['SiteTypesWithWeights'], 
                    DefaultWeight = 1.0 
                    )
        except KeyError:
            SiteTypesWithWeights = {AIParseUtils.VALUES:[], AIParseUtils.TOTAL_WEIGHT:0.0 }
            print "WARNING! IBIS generator: No SiteTypesWithWeights info specified in the OtherInfo field.\n\tSetting to default", SitesList
        ##print ">>>>>> SiteTypesWithWeightsDic", SiteTypesWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
        
        try:
            SitesWithWeightsDic = \
                AIParseUtils.readStringWithWeightsList( 
                    WLUnit['otherinfo']['SitesWithWeights'], 
                    DefaultWeight = 1.0 
                    )
        except KeyError:
            SitesWithWeightsDic = {AIParseUtils.VALUES:[], AIParseUtils.TOTAL_WEIGHT:0.0 }
            print "WARNING! IBIS generator: No SitesWithWeights info specified in the OtherInfo field.\n\tSetting to default", SitesList
        ##print ">>>>>> SitesWithWeightsDic", SitesWithWeightsDic[AIParseUtils.TOTAL_WEIGHT]
        
        try:    
            JavaMaxMemoryLimitMB = AIParseUtils.readInt(WLUnit['otherinfo']['JavaMaxMemoryLimitMB'], 10000)
        except KeyError:
            JavaMaxMemoryLimitMB = 10000
            print "WARNING! IBIS generator: No JavaMaxMemoryLimitMB info specified in the OtherInfo field.\n\tSetting to default", JavaMaxMemoryLimitMB, "MB"
        
        try:
            SelectionMethodName = WLUnit['otherinfo']['SelectionMethod']
        except KeyError:
            SelectionMethodName = 'R'
            raise Exception, "IBIS generator: No SelectionMethod info specified in the OtherInfo field."
            
    else:
        raise Exception, "IBIS generator: No OtherInfo data! Expected at least ComponentSize,NJobs,NComponents fields."
    
    UnitsDic = {}
    UnitsDic['info'] = {}
    UnitsDic['jobs'] = {}
    
    #-- init start time
    try:
        StartAt = AIParseUtils.readInt(WLUnit['otherinfo']['StartAt'], DefaultValue = 0)
        StartAt = StartAt * 1000 # the time is given is seconds
    except KeyError:
        StartAt = 0
    IBIS_LastRunTime = StartAt
    
    WLUnitMultiplicity = int(WLUnit['multiplicity'])
    if bGenerateRandom > 0:
        #--- generate random
        
        OneIBISSelectionMethod = IBISSelectionMethod( AppsWithWeightsDic )
        OneIBISSelectionMethod.setSelectionMethodByName( SelectionMethodName )
        
        JobIndex = 0
        if WLUnitMultiplicity > 1:
            NJobs = NJobs * WLUnitMultiplicity
        while JobIndex < NJobs:
            # 1. select application type
##            Application = AIRandomUtils.getRandomWeightedListElement(
##                       AppsWithWeightsDic,
##                       ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT 
##                    )
            Application = OneIBISSelectionMethod.getNextValue()
            if Application.find('*') >= 0:
                print "IBIS generator: Expanded", Application, "to", 
                Application = ExpandApplication(Application)
                print Application
            
            # 2. select the number of CPU for this job
            #TODO: repeat NCPUs selection until App can run on that many CPUs
            NCPUs = AIRandomUtils.getRandomWeightedListElement(
                       TotalCPUsWithWeightsDic,
                       ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT 
                    )
            # 3. select the number of components and try to create subcomponents,
            #    until successfully matching restrictions
            WhileCondition = 1
            NTries = 100
            while WhileCondition != 0 and NTries > 0:
                
                NTries = NTries - 1
                
                NComponents = AIRandomUtils.getRandomWeightedListElement(
                                    NComponentsWithWeightsDic,
                                    ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT 
                                    )
                                    
                if NComponents > NCPUs: 
                    continue # impossible match, just retry
                
                NCPUsPerComponent = NCPUs / NComponents
                # ensure restrictions are met 
                if NCPUsPerComponent < MinComponentSize or NCPUsPerComponent > MaxComponentSize:
                    continue
                
                NRemainingComponents = NCPUs - NCPUsPerComponent * NComponents
                if NRemainingComponents > 0 and EqualCPUsPerComponent == 1:
                    continue
                    
                # >= -> including the extra CPU for some components
                if NRemainingComponents > 0 and NCPUsPerComponent == MaxComponentSize:
                    continue
                
                SiteType = AIRandomUtils.getRandomWeightedListElement(
                                    SiteTypesWithWeightsDic,
                                    ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT 
                                    )
                
                PreComponentsList = []
                for ComponentID in xrange(NComponents):
                    
                    # cpu count
                    ToAssignHere = NCPUsPerComponent
                    if ComponentID < NRemainingComponents: # assign the extra CPUs, round-robin
                        ToAssignHere = ToAssignHere + 1
                        
                    # site location
                    if SiteType == 'ordered':
                        Site = AIRandomUtils.getRandomWeightedListElement(
                                    SitesWithWeightsDic,
                                    ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT 
                                    )
                    else:
                        Site = '*'
                        
                    # create pre-component information
                    PreComponent = {}
                    PreComponent['id'] = ComponentID
                    PreComponent['count'] = ToAssignHere
                    PreComponent['location'] = Site
                    
                    # add pre-component
                    PreComponentsList.append(PreComponent)
                    
                WhileCondition = 0
                    
            if WhileCondition == 1: # failed to generate in 100 tries
                continue # try again for the same job
            
            #--- generate rnd job
            #-- generate rnd job: ID
            CurrentUnitID = "%s-%d" % (UnitID, JobIndex % WLUnitMultiplicity)
            #-- generate rnd job: index
            UnitsDic['info'][JobIndex] = \
                        generateJobInfo( UnitDir, CurrentUnitID, JobIndex, Application, \
                                         WLUnit, SubmitDurationMS, Submitter )
            #-- generate rnd job: components
            UnitsDic['jobs'][JobIndex] = \
                        generateJobComponents( 
                            UnitDir, CurrentUnitID, JobIndex, WLUnit, 
                            PreComponentsList, JavaHomeDir, IbisAppsDir, IbisLibDir, 
                            JavaMaxMemoryLimitMB, Application, NCPUs
                            )
            #-- be more descriptive
            UnitsDic['info'][JobIndex]['description'] = \
                UnitsDic['info'][JobIndex]['description'] + \
                " Components: %d. NCPUs: %d." % (len(PreComponentsList), NCPUs) 
                            
            #--- done generating
            if UnitsDic['jobs'][JobIndex] == None:
                NJobs = NJobs - 1 #-- don't ack job, but make sure total no of jobs is decreased
            else:
                
                print "IBIS generator: Generated a job of type", Application, 
                print "with", len(PreComponentsList), "components/", 
                print NCPUs, "CPUs, in", UnitsDic['info'][JobIndex]['jdf']
                
                JobIndex = JobIndex + 1 #-- ack job 
                
                
        WLUnit['generatedjobs'] = JobIndex
        
    else:
        print "Complete workload"
        #--- generate all
        JobIndex = 0
        RepetitionIndex = 0
        while RepetitionIndex < WLUnitMultiplicity:
            for ComponentSize in ComponentSizeList:
                for NumComponents in NumComponentsList:
                    CurrentUnitID = "%s-%d" % (UnitID, JobIndex % WLUnitMultiplicity)
                    UnitsDic['info'][JobIndex] = \
                        generateJobInfo( UnitDir, CurrentUnitID, JobIndex, \
                                         WLUnit, SubmitDurationMS, Submitter )
                    UnitsDic['jobs'][JobIndex] = \
                        generateJobComponents( UnitDir, CurrentUnitID, JobIndex, WLUnit, \
                                               NumComponents, ComponentSize, Application, NCPUs )
                    JobIndex = JobIndex + 1
            RepetitionIndex = RepetitionIndex + 1
            
            if JobIndex > NJobs and NJobs > 0:
                print "Generated %d jobs and stopped because of the NJobs limit."
                break
            
        WLUnit['generatedjobs'] = JobIndex
        
    #raise Exception, "IBIS generator: Not implemented yet"
    return UnitsDic
예제 #15
0
        RunTime = IBIS_LastRunTime + WLUnit['arrivaltimefunc']( Name, ParamsList )
        IBIS_LastRunTime = RunTime
        InfoDic['runTime'] = "%.3f" % RunTime
    except AIRandomUtils.AIRandomError, e:
        print "AIRandomUtils.AIRandomError", e
        InfoDic['runTime'] = None
    except Exception, e:
        print "ERROR!", e, "Name", Name, "Params", ParamsList
        InfoDic['runTime'] = None
        
    if (InfoDic['runTime'] == None) or \
       (float(InfoDic['runTime']) < 0) or (float(InfoDic['runTime']) > SubmitDurationMS):
        # all workload units are started within at most SubmitDurationMS miliseconds
        print "Wrong arrival time of", InfoDic['runTime'], "generated by", Name, ParamsList
        print "Resetting to default U(0,%d)" % SubmitDurationMS
        InfoDic['runTime'] = str(AIRandomUtils.getRandomInt(0, SubmitDurationMS))
        
    return InfoDic

def generateJobComponents( UnitDir, UnitID, JobIndex, WLUnit, \
    PreComponentsList, JavaHomeDir, IbisAppsDir, IbisLibDir, \
    JavaMaxMemoryLimitMB, Application, NCPUs):
    
    # minutes: 3, 15, 30, 700
    RunTimeInMinutesList = [ '3', '15', '30', '700' ]
    
    NumComponents = len(PreComponentsList)
    
    ComponentsList = []
    
    #--- generate wall time once for all components of this job
예제 #16
0
                     ) 
         
         GraphData['NConnect'] = \
                 AIRandomUtils.getRandomWeightedListElement(
                     Params['TaskSamePred']['NConnectWithWeights'],
                     ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT 
                     ) 
     except Exception, e:
         print "Could not find a definition for", getRandomGraphTypeName(GraphType), "parameter", e
         continue
     GraphName = GraphName + ' NTasks=' + str(GraphData['NTasks']) + ' NConnect=' + str(GraphData['NConnect'])
 elif GraphType == RandomGraphType.LayerSameProb:
     try:
         GraphData['NLayers'] = \
                 AIRandomUtils.getRandomWeightedListElement(
                     Params['LayerSameProb']['NLayersWithWeights'],
                     ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT 
                     ) 
         GraphData['NLayerTasks'] = \
                 AIRandomUtils.getRandomWeightedListElement(
                     Params['LayerSameProb']['NLayerTasksWithWeights'],
                     ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT 
                     ) 
         GraphData['p'] = \
                 AIRandomUtils.getRandomWeightedListElement(
                     Params['LayerSameProb']['pWithWeights'],
                     ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT 
                     ) 
     except Exception, e:
         print "Could not find a definition for", getRandomGraphTypeName(GraphType), "parameter", e
         continue
     GraphName = GraphName + ' NLayers=' + str(GraphData['NLayers']) + ' NLayerTasks=' + str(GraphData['NLayerTasks']) + ' p=' + str(GraphData['p'])
예제 #17
0
def generateRandomStructures(WLOutDir, WLUnitID, WLUnit, SubmitDurationMS):
    #--- params read
    ##print ">>> WLUnit=", WLUnit
    try:
        NCompositeItems = AIParseUtils.readInt(WLUnit['otherinfo']['NCompositeItems'], 1)
    except:
        NCompositeItems = 1
        
    try:
        GraphTypesWithWeights = \
                AIParseUtils.readStringWithWeightsList( 
                    WLUnit['otherinfo']['GraphTypesWithWeights'], 
                    DefaultWeight = 1.0, ItemSeparator = ';' 
                    )
##        ValidGraphTypesWithWeights = {}
##        for GraphType in GraphTypesWithWeights[AIParseUtils.VALUES]:
##            if GraphTypes not in RandomGraphType.Names:
##                print "WARNING!", GraphTypes, "is not a valid GraphType. Skipping it."
##            else:
##                ValidGraphTypesWithWeights[GraphTypes] = GraphTypesWithWeights[AIParseUtils.VALUES][GraphTypes]
##        GraphTypesWithWeights[AIParseUtils.VALUES] = ValidGraphTypesWithWeights
    except:
        print "ERROR! No GraphTypesWithWeights info specified in the OtherInfo field.\n\tQuitting!"
        return None
    ##print ">>> GraphTypesWithWeights=", GraphTypesWithWeights
        
        
    STGFileNamesList = []
    try:
        STGFilters = WLUnit['otherinfo']['STGFiles']
        STGFilterList = STGFilters.split(',')
        for Filter in STGFilterList:
            CurrentSTGList = glob.glob(Filter)
            for STGFileName in CurrentSTGList:
                if STGFileName not in STGFileNamesList:
                    STGFileNamesList.append(STGFileName)
    except:
        STGFileNamesList = []
        pass
    
    TextGraphFileNamesList = []
    try:
        TextGraphFilters = WLUnit['otherinfo']['TextGraphFiles']
        TextGraphFilterList = TextGraphFilters.split(',')
        for Filter in TextGraphFilterList:
            CurrentTextGraphList = glob.glob(Filter)
            for TextGraphFileName in CurrentTextGraphList:
                if TextGraphFileName not in TextGraphFileNamesList:
                    TextGraphFileNamesList.append(TextGraphFileName)
                    #print "#### appending: " + TextGraphFileName
    except:
        TextGraphFileNamesList = []
        #print "### TextGraph exception"
        #print traceback.print_exc()
        pass
        
    ##print ">>>", "before Params = readAllParams( WLUnit )"
    Params = readAllParams( WLUnit )
    ##print ">>>", "after Params = readAllParams( WLUnit )"
        
    #--- init
    ListOfGraphs = []
        
    #--- generate all composite structures
    index = 0
    maxIndex = NCompositeItems
    while index < maxIndex:
        #--- generate one composite structure
        GraphID = WLUnitID + ("-%d_graph" % index)
        #print("### Graph ID initial: " + GraphID)
        #print ("### WLUnit ID initial: " + WLUnitID)
        GraphType = AIRandomUtils.getRandomWeightedListElement(
                            GraphTypesWithWeights,
                            ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT 
                            )
        GraphType = RandomGraphType.Names[GraphType]
        #print ">>>", "Selected", GraphType, "from", GraphTypesWithWeights
        
        
        GraphName = getRandomGraphTypeName(GraphType)
        GraphData = {}
        if GraphType == RandomGraphType.FromSTG:
            pass            
        elif GraphType == RandomGraphType.TaskSameProb:
            try:
                GraphData['NTasks'] = \
                        AIRandomUtils.getRandomWeightedListElement(
                            Params['TaskSameProb']['NTasksWithWeights'],
                            ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT 
                            ) 
                GraphData['p'] = \
                        AIRandomUtils.getRandomWeightedListElement(
                            Params['TaskSameProb']['pWithWeights'],
                            ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT 
                            ) 
            except Exception, e:
                print "Could not find a definition for", getRandomGraphTypeName(GraphType), "parameter", e
                continue
            GraphName = GraphName + ' NTasks=' + str(GraphData['NTasks']) + ' p=' + str(GraphData['p'])
        elif GraphType == RandomGraphType.TaskSamePred:
            try:
                GraphData['NTasks'] = \
                        AIRandomUtils.getRandomWeightedListElement(
                            Params['TaskSamePred']['NTasksWithWeights'],
                            ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT 
                            ) 
                
                GraphData['NConnect'] = \
                        AIRandomUtils.getRandomWeightedListElement(
                            Params['TaskSamePred']['NConnectWithWeights'],
                            ValuesKey = AIParseUtils.VALUES, TotalWeightKey = AIParseUtils.TOTAL_WEIGHT 
                            ) 
            except Exception, e:
                print "Could not find a definition for", getRandomGraphTypeName(GraphType), "parameter", e
                continue
예제 #18
0
    except AIRandomUtils.AIRandomError, e:
        print "AIRandomUtils.AIRandomError", e
        InfoDic["runTime"] = None
    except Exception, e:
        print "ERROR!", e, "Name", Name, "Params", ParamsList
        InfoDic["runTime"] = None

    if (
        (InfoDic["runTime"] == None)
        or (float(InfoDic["runTime"]) < 0)
        or (float(InfoDic["runTime"]) > SubmitDurationMS)
    ):
        # all workload units are started within at most SubmitDurationMS miliseconds
        print "Wrong arrival time of", InfoDic["runTime"], "generated by", Name, ParamsList
        print "Resetting to default U(0,%d)" % SubmitDurationMS
        InfoDic["runTime"] = str(AIRandomUtils.getRandomInt(0, SubmitDurationMS))

    return InfoDic


def generateJobComponents(UnitDir, UnitID, JobIndex, WLUnit, NumComponents, ComponentSize, Application):

    # minutes: 3, 15, 30, 700
    RunTimeInMinutesList = ["3", "15", "30", "700"]

    ComponentsList = []

    # --- generate wall time once for all components of this job
    MaxWallTime = AIRandomUtils.getRandomListElement(RunTimeInMinutesList)

    if UnitDir[0] != "/":